From b5791773f3c46a79128c4bae6fbf6226074fa9ae Mon Sep 17 00:00:00 2001 From: Arie Date: Fri, 7 Nov 2014 15:44:20 -0800 Subject: [PATCH 001/732] Initial commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000000..157f813e551a --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +git-demo +======== From d5ae1bf0f98567ea46b39abb7805094a7a054ac0 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 7 Nov 2014 15:59:08 -0800 Subject: [PATCH 002/732] initial --- .checkstyle | 7 ++++ .classpath | 36 +++++++++++++++++++ .gitignore | 1 + .project | 29 +++++++++++++++ .settings/org.eclipse.jdt.core.prefs | 13 +++++++ .settings/org.eclipse.m2e.core.prefs | 4 +++ pom.xml | 6 ++++ .../gcloud/datastore/DatastoreService.java | 5 +++ 8 files changed, 101 insertions(+) create mode 100644 .checkstyle create mode 100644 .classpath create mode 100644 .gitignore create mode 100644 .project create mode 100644 .settings/org.eclipse.jdt.core.prefs create mode 100644 .settings/org.eclipse.m2e.core.prefs create mode 100644 pom.xml create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreService.java diff --git a/.checkstyle b/.checkstyle new file mode 100644 index 000000000000..5783bc0d77a1 --- /dev/null +++ b/.checkstyle @@ -0,0 +1,7 @@ + + + + + + + diff --git a/.classpath b/.classpath new file mode 100644 index 000000000000..9fc2de7b0d03 --- /dev/null +++ b/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..b83d22266ac8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/.project b/.project new file mode 100644 index 000000000000..c8b8fcbb843d --- /dev/null +++ b/.project @@ -0,0 +1,29 @@ + + + git-demo + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + net.sf.eclipsecs.core.CheckstyleBuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + net.sf.eclipsecs.core.CheckstyleNature + + diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000000..60ec3468d009 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,13 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.5 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.5 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 000000000000..f897a7f1cb23 --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000000..6898b1fa6b19 --- /dev/null +++ b/pom.xml @@ -0,0 +1,6 @@ + + 4.0.0 + com.ozarov.testing + git-demo + 0.0.1-SNAPSHOT + \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java new file mode 100644 index 000000000000..549fa0c0366f --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -0,0 +1,5 @@ +package com.google.gcloud.datastore; + +public interface DatastoreService { + +} From 5115b31a5d1a0d4a56b0d222bd66a3035cbb7650 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 7 Nov 2014 16:06:18 -0800 Subject: [PATCH 003/732] add dummy method --- src/main/java/com/google/gcloud/datastore/DatastoreService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 549fa0c0366f..0ebf1b682608 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -2,4 +2,6 @@ public interface DatastoreService { + String getSchema(); + } From c7bac98fa8c75285c711acc3fa6870b69580553b Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 12 Nov 2014 09:41:10 -0800 Subject: [PATCH 004/732] Adding service options --- pom.xml | 12 ++++++++++++ .../google/gcloud/datastore/DatastoreService.java | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6898b1fa6b19..bc96c067d258 100644 --- a/pom.xml +++ b/pom.xml @@ -3,4 +3,16 @@ com.ozarov.testing git-demo 0.0.1-SNAPSHOT + + + com.google.http-client + google-http-client + 1.18.0-rc + + + com.google.oauth-client + google-oauth-client + 1.18.0-rc + + \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 0ebf1b682608..2121d2daa0c1 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -2,6 +2,6 @@ public interface DatastoreService { - String getSchema(); + String get(String key) } From 25fb54375270663755aa43e99103943f6e93fd9b Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 12 Nov 2014 09:48:28 -0800 Subject: [PATCH 005/732] Adding service options and factory --- .../com/google/gcloud/ServiceOptions.java | 116 +++++++++++++++++ .../datastore/DatastoreServiceFactory.java | 7 ++ .../datastore/DatastoreServiceOptions.java | 118 ++++++++++++++++++ 3 files changed, 241 insertions(+) create mode 100644 src/main/java/com/google/gcloud/ServiceOptions.java create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java new file mode 100644 index 000000000000..80224cd1ca3f --- /dev/null +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -0,0 +1,116 @@ +package com.google.gcloud; + +import java.util.Arrays; +import java.util.List; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; + +public class ServiceOptions { + + private final String host; + private static final String DEFAULT_HOST = "https://www.googleapis.com"; + private final HttpRequestInitializer initializer; + private final Credential credential; + private final HttpTransport transport; + + public static final List SCOPES = Arrays.asList( + "https://www.googleapis.com/auth/datastore", + "https://www.googleapis.com/auth/userinfo.email"); + + ServiceOptions(Builder b) { + this.dataset = b.dataset; + this.host = b.host != null ? b.host : DEFAULT_HOST; + this.initializer = b.initializer; + this.credential = b.credential; + this.transport = b.transport; + } + + /** + * Builder for {@link ServiceOptions}. + */ + protected static class Builder { + private String dataset; + private String host; + private HttpRequestInitializer initializer; + private Credential credential; + private HttpTransport transport; + + public Builder() { } + + public Builder(ServiceOptions options) { + this.dataset = options.dataset; + this.host = options.host; + this.initializer = options.initializer; + this.credential = options.credential; + this.transport = options.transport; + } + + public ServiceOptions build() { + return new ServiceOptions(this); + } + + /** + * Sets the dataset used to access the datastore. + */ + public Builder dataset(String newDataset) { + dataset = newDataset; + return this; + } + + /** + * Sets the host used to access the datastore. + */ + public Builder host(String newHost) { + host = newHost; + return this; + } + + /** + * Sets the (optional) initializer to run on HTTP requests to the API. + */ + public Builder initializer(HttpRequestInitializer newInitializer) { + initializer = newInitializer; + return this; + } + + /** + * Sets the Google APIs credentials used to access the API. + */ + public Builder credential(Credential newCredential) { + credential = newCredential; + return this; + } + + /** + * Sets the transport used to access the API. + */ + public Builder transport(HttpTransport transport) { + this.transport = transport; + return this; + } + } + + // === getters === + + public String getDataset() { + return dataset; + } + + public String getHost() { + return host; + } + + public HttpRequestInitializer getInitializer() { + return initializer; + } + + public Credential getCredential() { + return credential; + } + + public HttpTransport getTransport() { + return transport; + } +} \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java new file mode 100644 index 000000000000..aca1075fae64 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java @@ -0,0 +1,7 @@ +package com.google.gcloud.datastore; + + +public interface DatastoreServiceFactory { + + DatastoreService getDatastoreService(DatastoreServiceOptions options); +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java new file mode 100644 index 000000000000..981d90515196 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -0,0 +1,118 @@ +package com.google.gcloud.datastore; + +import java.util.Arrays; +import java.util.List; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.gcloud.ServiceOptions; + +public class DatastoreServiceOptions extends ServiceOptions { + private final String dataset; + + private static final String DEFAULT_HOST = "https://www.googleapis.com"; + + private final HttpRequestInitializer initializer; + + private final Credential credential; + private final HttpTransport transport; + public static final List SCOPES = Arrays.asList( + "https://www.googleapis.com/auth/datastore", + "https://www.googleapis.com/auth/userinfo.email"); + + DatastoreServiceOptions(Builder b) { + this.dataset = b.dataset; + this.host = b.host != null ? b.host : DEFAULT_HOST; + this.initializer = b.initializer; + this.credential = b.credential; + this.transport = b.transport; + } + + /** + * Builder for {@link DatastoreServiceOptions}. + */ + public static class Builder { + private String dataset; + private String host; + private HttpRequestInitializer initializer; + private Credential credential; + private HttpTransport transport; + + public Builder() { } + + public Builder(DatastoreServiceOptions options) { + this.dataset = options.dataset; + this.host = options.host; + this.initializer = options.initializer; + this.credential = options.credential; + this.transport = options.transport; + } + + public DatastoreServiceOptions build() { + return new DatastoreServiceOptions(this); + } + + /** + * Sets the dataset used to access the datastore. + */ + public Builder dataset(String newDataset) { + dataset = newDataset; + return this; + } + + /** + * Sets the host used to access the datastore. + */ + public Builder host(String newHost) { + host = newHost; + return this; + } + + /** + * Sets the (optional) initializer to run on HTTP requests to the API. + */ + public Builder initializer(HttpRequestInitializer newInitializer) { + initializer = newInitializer; + return this; + } + + /** + * Sets the Google APIs credentials used to access the API. + */ + public Builder credential(Credential newCredential) { + credential = newCredential; + return this; + } + + /** + * Sets the transport used to access the API. + */ + public Builder transport(HttpTransport transport) { + this.transport = transport; + return this; + } + } + + // === getters === + + public String getDataset() { + return dataset; + } + + public String getHost() { + return host; + } + + public HttpRequestInitializer getInitializer() { + return initializer; + } + + public Credential getCredential() { + return credential; + } + + public HttpTransport getTransport() { + return transport; + } +} \ No newline at end of file From 68d808ac1cd8a032cf28adf2077df92d87e1cd12 Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 12 Nov 2014 17:01:22 -0800 Subject: [PATCH 006/732] work in progress --- .classpath | 20 +--- .project | 8 +- .settings/org.eclipse.jdt.core.prefs | 98 ++++++++++++++-- pom.xml | 5 + .../com/google/gcloud/ServiceOptions.java | 102 +++------------- .../gcloud/datastore/DatastoreService.java | 2 +- .../datastore/DatastoreServiceFactory.java | 6 +- .../datastore/DatastoreServiceOptions.java | 110 +++--------------- 8 files changed, 149 insertions(+), 202 deletions(-) diff --git a/.classpath b/.classpath index 9fc2de7b0d03..86bbd24e2c1d 100644 --- a/.classpath +++ b/.classpath @@ -6,31 +6,21 @@ - - - - - - - - - - - - - - - + + + + + diff --git a/.project b/.project index c8b8fcbb843d..47ba158d4646 100644 --- a/.project +++ b/.project @@ -10,13 +10,18 @@ + + net.sf.eclipsecs.core.CheckstyleBuilder + + + org.eclipse.m2e.core.maven2Builder - net.sf.eclipsecs.core.CheckstyleBuilder + edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder @@ -25,5 +30,6 @@ org.eclipse.jdt.core.javanature org.eclipse.m2e.core.maven2Nature net.sf.eclipsecs.core.CheckstyleNature + edu.umd.cs.findbugs.plugin.eclipse.findbugsNature diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 60ec3468d009..2ffb1f42da80 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,13 +1,95 @@ eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.5 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=error +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=warning org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=warning +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=warning +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.5 +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=warning +org.eclipse.jdt.core.compiler.problem.localVariableHiding=warning +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=warning +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=warning +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=warning +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=warning +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=warning +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=error +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=warning +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=error +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=warning +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=warning +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=warning +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=disabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=warning +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=warning +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=warning +org.eclipse.jdt.core.compiler.problem.unusedParameter=warning +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=warning +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/pom.xml b/pom.xml index bc96c067d258..3e0d202f7677 100644 --- a/pom.xml +++ b/pom.xml @@ -14,5 +14,10 @@ google-oauth-client 1.18.0-rc + + com.google.guava + guava + RELEASE + \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 80224cd1ca3f..a435dbefb071 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -1,116 +1,50 @@ package com.google.gcloud; -import java.util.Arrays; -import java.util.List; - import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.http.HttpTransport; +import com.google.common.base.MoreObjects; -public class ServiceOptions { +public abstract class ServiceOptions { - private final String host; private static final String DEFAULT_HOST = "https://www.googleapis.com"; - private final HttpRequestInitializer initializer; - private final Credential credential; - private final HttpTransport transport; - public static final List SCOPES = Arrays.asList( - "https://www.googleapis.com/auth/datastore", - "https://www.googleapis.com/auth/userinfo.email"); + private final String host; + private final Credential credential; - ServiceOptions(Builder b) { - this.dataset = b.dataset; - this.host = b.host != null ? b.host : DEFAULT_HOST; - this.initializer = b.initializer; - this.credential = b.credential; - this.transport = b.transport; + protected ServiceOptions(Builder builder) { + host = MoreObjects.firstNonNull(builder.host, DEFAULT_HOST); + credential = builder.credential; } - /** - * Builder for {@link ServiceOptions}. - */ - protected static class Builder { - private String dataset; + protected abstract static class Builder { + private String host; - private HttpRequestInitializer initializer; private Credential credential; - private HttpTransport transport; - public Builder() { } + public Builder() {} public Builder(ServiceOptions options) { - this.dataset = options.dataset; - this.host = options.host; - this.initializer = options.initializer; - this.credential = options.credential; - this.transport = options.transport; + host = options.host; + credential = options.credential; } - public ServiceOptions build() { - return new ServiceOptions(this); - } - - /** - * Sets the dataset used to access the datastore. - */ - public Builder dataset(String newDataset) { - dataset = newDataset; - return this; - } + protected abstract ServiceOptions build(); - /** - * Sets the host used to access the datastore. - */ - public Builder host(String newHost) { - host = newHost; + public Builder setHost(String host) { + this.host = host; return this; } - /** - * Sets the (optional) initializer to run on HTTP requests to the API. - */ - public Builder initializer(HttpRequestInitializer newInitializer) { - initializer = newInitializer; - return this; - } - - /** - * Sets the Google APIs credentials used to access the API. - */ - public Builder credential(Credential newCredential) { - credential = newCredential; - return this; - } - - /** - * Sets the transport used to access the API. - */ - public Builder transport(HttpTransport transport) { - this.transport = transport; + public Builder setCredential(Credential credential) { + this.credential = credential; return this; } } - // === getters === - - public String getDataset() { - return dataset; - } - public String getHost() { return host; } - public HttpRequestInitializer getInitializer() { - return initializer; - } - public Credential getCredential() { return credential; } - - public HttpTransport getTransport() { - return transport; - } -} \ No newline at end of file +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 2121d2daa0c1..0923f42c5b19 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -2,6 +2,6 @@ public interface DatastoreService { - String get(String key) + String get(String key); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java index aca1075fae64..903e33df5bf9 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java @@ -1,7 +1,9 @@ package com.google.gcloud.datastore; -public interface DatastoreServiceFactory { +public class DatastoreServiceFactory { - DatastoreService getDatastoreService(DatastoreServiceOptions options); + public DatastoreService getDatastoreService(DatastoreServiceOptions options) { + return new DatastoreServiceImpl(options); + } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 981d90515196..13315ea62e7a 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -1,118 +1,46 @@ package com.google.gcloud.datastore; -import java.util.Arrays; -import java.util.List; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.http.HttpTransport; +import com.google.common.collect.ImmutableList; import com.google.gcloud.ServiceOptions; -public class DatastoreServiceOptions extends ServiceOptions { - private final String dataset; +import java.util.List; - private static final String DEFAULT_HOST = "https://www.googleapis.com"; +public class DatastoreServiceOptions extends ServiceOptions { - private final HttpRequestInitializer initializer; + private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore"; + private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email"; + private static final List SCOPES = ImmutableList.of(DATASTORE_SCOPE, USERINFO_SCOPE); - private final Credential credential; - private final HttpTransport transport; - public static final List SCOPES = Arrays.asList( - "https://www.googleapis.com/auth/datastore", - "https://www.googleapis.com/auth/userinfo.email"); + private final String dataset; - DatastoreServiceOptions(Builder b) { - this.dataset = b.dataset; - this.host = b.host != null ? b.host : DEFAULT_HOST; - this.initializer = b.initializer; - this.credential = b.credential; - this.transport = b.transport; + DatastoreServiceOptions(Builder builder) { + super(builder); + dataset = builder.dataset; } - /** - * Builder for {@link DatastoreServiceOptions}. - */ - public static class Builder { + public static class Builder extends ServiceOptions.Builder { + private String dataset; - private String host; - private HttpRequestInitializer initializer; - private Credential credential; - private HttpTransport transport; - public Builder() { } + public Builder() {} public Builder(DatastoreServiceOptions options) { - this.dataset = options.dataset; - this.host = options.host; - this.initializer = options.initializer; - this.credential = options.credential; - this.transport = options.transport; + super(options); + dataset = options.dataset; } + @Override public DatastoreServiceOptions build() { return new DatastoreServiceOptions(this); } - /** - * Sets the dataset used to access the datastore. - */ - public Builder dataset(String newDataset) { - dataset = newDataset; - return this; - } - - /** - * Sets the host used to access the datastore. - */ - public Builder host(String newHost) { - host = newHost; - return this; - } - - /** - * Sets the (optional) initializer to run on HTTP requests to the API. - */ - public Builder initializer(HttpRequestInitializer newInitializer) { - initializer = newInitializer; - return this; - } - - /** - * Sets the Google APIs credentials used to access the API. - */ - public Builder credential(Credential newCredential) { - credential = newCredential; - return this; - } - - /** - * Sets the transport used to access the API. - */ - public Builder transport(HttpTransport transport) { - this.transport = transport; + public Builder setDataset(String dataset) { + this.dataset = dataset; return this; } } - // === getters === - public String getDataset() { return dataset; } - - public String getHost() { - return host; - } - - public HttpRequestInitializer getInitializer() { - return initializer; - } - - public Credential getCredential() { - return credential; - } - - public HttpTransport getTransport() { - return transport; - } -} \ No newline at end of file +} From ef72daa81c73f99e2156f5bfe8127591fc6358e9 Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 12 Nov 2014 22:36:12 -0800 Subject: [PATCH 007/732] work in progress --- pom.xml | 7 ++ .../com/google/gcloud/ServiceOptions.java | 75 ++++++++++++++++--- .../datastore/DatastoreServiceImpl.java | 17 +++++ .../datastore/DatastoreServiceOptions.java | 6 +- 4 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java diff --git a/pom.xml b/pom.xml index 3e0d202f7677..318739824c67 100644 --- a/pom.xml +++ b/pom.xml @@ -19,5 +19,12 @@ guava RELEASE + + com.google.apis + + google-api-services-datastore-protobuf + + v1beta2-rev1-2.1.0 + \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index a435dbefb071..325daf08ce37 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -1,30 +1,60 @@ package com.google.gcloud; -import com.google.api.client.auth.oauth2.Credential; + +import com.google.api.client.googleapis.compute.ComputeCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson.JacksonFactory; import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.PrivateKey; +import java.util.Set; public abstract class ServiceOptions { private static final String DEFAULT_HOST = "https://www.googleapis.com"; private final String host; - private final Credential credential; + private final HttpTransport httpTransport; + private final HttpRequestInitializer httpRequestInitializer; protected ServiceOptions(Builder builder) { host = MoreObjects.firstNonNull(builder.host, DEFAULT_HOST); - credential = builder.credential; + httpTransport = MoreObjects.firstNonNull(builder.httpTransport, getDefaultHttpTransport()); + httpRequestInitializer = builder.httpRequestInitializer; + } + + private static HttpTransport getDefaultHttpTransport() { + try { + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + // Try to connect using Google Compute Engine service account credentials. + ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); + // Force token refresh to detect if we are running on Google Compute Engine. + credential.refreshToken(); + return credential.getTransport(); + } catch (IOException | GeneralSecurityException e) { + return new NetHttpTransport(); + } } protected abstract static class Builder { private String host; - private Credential credential; + private HttpTransport httpTransport; + private HttpRequestInitializer httpRequestInitializer; + private PrivateKey privateKey; public Builder() {} - public Builder(ServiceOptions options) { + protected Builder(ServiceOptions options) { host = options.host; - credential = options.credential; + httpTransport = options.httpTransport; + httpRequestInitializer = options.httpRequestInitializer; } protected abstract ServiceOptions build(); @@ -34,17 +64,42 @@ public Builder setHost(String host) { return this; } - public Builder setCredential(Credential credential) { - this.credential = credential; + public Builder setHttpTransport(HttpTransport httpTransport) { + this.httpTransport = httpTransport; + return this; + } + + public Builder setHttpRequestInitializer(HttpRequestInitializer httpRequestInitializer) { + // TODO: replace HttpRequestInitializer with CrendentialProvider - 2 subclasses + // one that is set with HttpRequestInitializer (and another that is set + // with both private key and service account) + // Also, consider instead of HttpRequestIntializer option to have "AppEngine" option + // which will use reflection to create HttpRequestInitializer + Preconditions.checkArgument( + privateKey == null, "Can't set both PrivateKey and HttpRequestInitializer"); + this.httpRequestInitializer = httpRequestInitializer; + return this; + } + + public Builder setPrivateKey(PrivateKey privateKey) { + Preconditions.checkArgument( + httpRequestInitializer == null, "Can't set both PrivateKey and HttpRequestInitializer"); + this.privateKey = privateKey; return this; } } + protected abstract Set getScopes(); + public String getHost() { return host; } - public Credential getCredential() { - return credential; + public HttpTransport getHttpTransport() { + return httpTransport; + } + + public HttpRequestInitializer getHttpRequestInitializer() { + return httpRequestInitializer; } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java new file mode 100644 index 000000000000..4793436fd6f0 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -0,0 +1,17 @@ +package com.google.gcloud.datastore; + +final class DatastoreServiceImpl implements DatastoreService { + + private final DatastoreServiceOptions options; + + DatastoreServiceImpl(DatastoreServiceOptions options) { + this.options = options; + } + + @Override + public String get(String key) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 13315ea62e7a..065918d69ac3 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -1,15 +1,15 @@ package com.google.gcloud.datastore; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.gcloud.ServiceOptions; -import java.util.List; +import java.util.Set; public class DatastoreServiceOptions extends ServiceOptions { private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore"; private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email"; - private static final List SCOPES = ImmutableList.of(DATASTORE_SCOPE, USERINFO_SCOPE); + private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE); private final String dataset; From 22aa45fb27d68d0a035835a0e07a00784697f397 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 13 Nov 2014 12:59:51 -0800 Subject: [PATCH 008/732] dummy --- .../de.loskutov.anyedit.AnyEditTools.prefs | 18 ++++++++++++++++++ .settings/org.eclipse.jdt.core.prefs | 12 +++--------- 2 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 .settings/de.loskutov.anyedit.AnyEditTools.prefs diff --git a/.settings/de.loskutov.anyedit.AnyEditTools.prefs b/.settings/de.loskutov.anyedit.AnyEditTools.prefs new file mode 100644 index 000000000000..c9ca2a972d8a --- /dev/null +++ b/.settings/de.loskutov.anyedit.AnyEditTools.prefs @@ -0,0 +1,18 @@ +activeContentFilterList=*.makefile,makefile,*.Makefile,Makefile,Makefile.*,*.mk,MANIFEST.MF,.project +addNewLine=true +convertActionOnSaave=AnyEdit.CnvrtTabToSpaces +eclipse.preferences.version=1 +fixLineDelimiters=false +ignoreBlankLinesWhenTrimming=false +inActiveContentFilterList= +javaTabWidthForJava=true +org.eclipse.jdt.ui.editor.tab.width=2 +projectPropsEnabled=false +removeTrailingSpaces=true +replaceAllSpaces=false +replaceAllTabs=false +saveAndAddLine=false +saveAndConvert=false +saveAndFixLineDelimiters=false +saveAndTrim=true +useModulo4Tabs=false diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 60ec3468d009..f42de363afaa 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,13 +1,7 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.5 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.5 +org.eclipse.jdt.core.compiler.source=1.7 From 468d80ae3cf199683c64ae16053f890178c8c1bf Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 13 Nov 2014 13:00:07 -0800 Subject: [PATCH 009/732] dummy --- .classpath | 10 +++++----- .project | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.classpath b/.classpath index 9fc2de7b0d03..e177ae03a45e 100644 --- a/.classpath +++ b/.classpath @@ -1,5 +1,10 @@ + + + + + @@ -22,11 +27,6 @@ - - - - - diff --git a/.project b/.project index c8b8fcbb843d..9a5588b19d87 100644 --- a/.project +++ b/.project @@ -20,10 +20,16 @@ + + edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder + + + org.eclipse.jdt.core.javanature org.eclipse.m2e.core.maven2Nature net.sf.eclipsecs.core.CheckstyleNature + edu.umd.cs.findbugs.plugin.eclipse.findbugsNature From 40380c5c9e66fbebc9162a153c0b26f8ae9a7006 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 14 Nov 2014 18:01:43 -0800 Subject: [PATCH 010/732] work in progress --- pom.xml | 5 ++ .../java/com/google/gcloud/AuthConfig.java | 57 +++++++++++++ .../com/google/gcloud/ServiceOptions.java | 83 ++++++++++++------- .../datastore/DatastoreServiceOptions.java | 5 ++ 4 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 src/main/java/com/google/gcloud/AuthConfig.java diff --git a/pom.xml b/pom.xml index 318739824c67..534629a90885 100644 --- a/pom.xml +++ b/pom.xml @@ -26,5 +26,10 @@ v1beta2-rev1-2.1.0 + + com.google.api-client + google-api-client-appengine + 1.18.0-rc + \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java new file mode 100644 index 000000000000..2bf34ae29ef8 --- /dev/null +++ b/src/main/java/com/google/gcloud/AuthConfig.java @@ -0,0 +1,57 @@ +package com.google.gcloud; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.jackson.JacksonFactory; + +import java.security.PrivateKey; +import java.util.Set; + +public abstract class AuthConfig { + + protected abstract HttpRequestInitializer getHttpRequestInitializer( + HttpTransport transport, Set scopes); + + + public static AuthConfig createForAppEngine() { + return new AppEngineAuthConfig(); + } + + public static AuthConfig createForAccount(String account, PrivateKey privateKey) { + return new ServiceAccountAuthConfig(account, privateKey); + } + + private static class AppEngineAuthConfig extends AuthConfig { + + @Override + protected HttpRequestInitializer getHttpRequestInitializer( + HttpTransport transport, Set scopes) { + return new AppIdentityCredential(scopes); + } + } + + private static class ServiceAccountAuthConfig extends AuthConfig { + + private final String account; + private final PrivateKey privateKey; + + public ServiceAccountAuthConfig(String account, PrivateKey privateKey) { + this.account = account; + this.privateKey = privateKey; + } + + @Override + protected HttpRequestInitializer getHttpRequestInitializer( + HttpTransport transport, Set scopes) { + return new GoogleCredential.Builder() + .setTransport(transport) + .setJsonFactory(new JacksonFactory()) + .setServiceAccountId(account) + .setServiceAccountScopes(scopes) + .setServiceAccountPrivateKey(privateKey) + .build(); + } + } +} diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 325daf08ce37..c924bef6be3c 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -1,6 +1,7 @@ package com.google.gcloud; +import com.google.api.client.extensions.appengine.http.UrlFetchTransport; import com.google.api.client.googleapis.compute.ComputeCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.http.HttpRequestInitializer; @@ -8,11 +9,9 @@ import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.jackson.JacksonFactory; import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; import java.io.IOException; import java.security.GeneralSecurityException; -import java.security.PrivateKey; import java.util.Set; public abstract class ServiceOptions { @@ -21,40 +20,76 @@ public abstract class ServiceOptions { private final String host; private final HttpTransport httpTransport; - private final HttpRequestInitializer httpRequestInitializer; + private final AuthConfig authConfig; protected ServiceOptions(Builder builder) { host = MoreObjects.firstNonNull(builder.host, DEFAULT_HOST); httpTransport = MoreObjects.firstNonNull(builder.httpTransport, getDefaultHttpTransport()); - httpRequestInitializer = builder.httpRequestInitializer; + authConfig = MoreObjects.firstNonNull(builder.authConfig, getDefaultAuthConfig()); } private static HttpTransport getDefaultHttpTransport() { + // Consider App Engine + if (System.getProperty("com.google.appengine.application.id") != null) { + try { + return new UrlFetchTransport(); + } catch (Exception ignore) { + // Maybe not on App Engine + } + } + // Consider Compute try { - NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); - // Try to connect using Google Compute Engine service account credentials. - ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); - // Force token refresh to detect if we are running on Google Compute Engine. - credential.refreshToken(); - return credential.getTransport(); + return getComputeCredential().getTransport(); } catch (IOException | GeneralSecurityException e) { return new NetHttpTransport(); } } + private static AuthConfig getDefaultAuthConfig() { + // Consider App Engine + if (System.getProperty("com.google.appengine.application.id") != null) { + try { + return AuthConfig.createForAppEngine(); + } catch (Exception ignore) { + // Maybe not on App Engine + } + } + // Consider Compute + try { + final ComputeCredential cred = getComputeCredential(); + return new AuthConfig() { + @Override protected HttpRequestInitializer getHttpRequestInitializer( + HttpTransport transport, Set scopes) { + return cred; + } + }; + } catch (IOException | GeneralSecurityException e) { + return AuthConfig.createForAccount(null, null); + } + } + + private static ComputeCredential getComputeCredential() + throws IOException, GeneralSecurityException { + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + // Try to connect using Google Compute Engine service account credentials. + ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); + // Force token refresh to detect if we are running on Google Compute Engine. + credential.refreshToken(); + return credential; + } + protected abstract static class Builder { private String host; private HttpTransport httpTransport; - private HttpRequestInitializer httpRequestInitializer; - private PrivateKey privateKey; + private AuthConfig authConfig; public Builder() {} protected Builder(ServiceOptions options) { host = options.host; httpTransport = options.httpTransport; - httpRequestInitializer = options.httpRequestInitializer; + authConfig = options.authConfig; } protected abstract ServiceOptions build(); @@ -69,22 +104,8 @@ public Builder setHttpTransport(HttpTransport httpTransport) { return this; } - public Builder setHttpRequestInitializer(HttpRequestInitializer httpRequestInitializer) { - // TODO: replace HttpRequestInitializer with CrendentialProvider - 2 subclasses - // one that is set with HttpRequestInitializer (and another that is set - // with both private key and service account) - // Also, consider instead of HttpRequestIntializer option to have "AppEngine" option - // which will use reflection to create HttpRequestInitializer - Preconditions.checkArgument( - privateKey == null, "Can't set both PrivateKey and HttpRequestInitializer"); - this.httpRequestInitializer = httpRequestInitializer; - return this; - } - - public Builder setPrivateKey(PrivateKey privateKey) { - Preconditions.checkArgument( - httpRequestInitializer == null, "Can't set both PrivateKey and HttpRequestInitializer"); - this.privateKey = privateKey; + public Builder setAuthConfig(AuthConfig authConfig) { + this.authConfig = authConfig; return this; } } @@ -99,7 +120,7 @@ public HttpTransport getHttpTransport() { return httpTransport; } - public HttpRequestInitializer getHttpRequestInitializer() { - return httpRequestInitializer; + public AuthConfig getAuthConfig() { + return authConfig; } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 065918d69ac3..5ece5b39e378 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -43,4 +43,9 @@ public Builder setDataset(String dataset) { public String getDataset() { return dataset; } + + @Override + protected Set getScopes() { + return SCOPES; + } } From e2865eeea5e9095db886bc307e17abe42dc56e41 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 18 Nov 2014 16:35:18 -0800 Subject: [PATCH 011/732] work in progress - adding datastore --- .settings/org.eclipse.jdt.core.prefs | 103 +--------- .../org.eclipse.ltk.core.refactoring.prefs | 2 + .../com/google/gcloud/ServiceOptions.java | 27 ++- .../google/gcloud/datastore/CompleteKey.java | 94 +++++++++ .../gcloud/datastore/DatastoreService.java | 15 +- .../datastore/DatastoreServiceImpl.java | 6 - .../datastore/DatastoreServiceOptions.java | 24 ++- .../java/com/google/gcloud/datastore/Key.java | 183 ++++++++++++++++++ .../google/gcloud/datastore/KeyMapValue.java | 24 +++ .../google/gcloud/datastore/NullValue.java | 31 +++ .../google/gcloud/datastore/StringValue.java | 47 +++++ .../com/google/gcloud/datastore/Value.java | 129 ++++++++++++ 12 files changed, 565 insertions(+), 120 deletions(-) create mode 100644 .settings/org.eclipse.ltk.core.refactoring.prefs create mode 100644 src/main/java/com/google/gcloud/datastore/CompleteKey.java create mode 100644 src/main/java/com/google/gcloud/datastore/Key.java create mode 100644 src/main/java/com/google/gcloud/datastore/KeyMapValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/NullValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/StringValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/Value.java diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index afa74a52a006..0ba1de1e6e8a 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,101 +1,4 @@ +=\=\=\=\=\=\= +<<<<<<<=HEAD +>>>>>>>=ef72daa81c73f99e2156f5bfe8127591fc6358e9 eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled -org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore -org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull -org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault -org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable -org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.compliance=1.7 -<<<<<<< HEAD -======= -org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning ->>>>>>> ef72daa81c73f99e2156f5bfe8127591fc6358e9 -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.autoboxing=ignore -org.eclipse.jdt.core.compiler.problem.comparingIdentical=error -org.eclipse.jdt.core.compiler.problem.deadCode=warning -org.eclipse.jdt.core.compiler.problem.deprecation=warning -org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled -org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled -org.eclipse.jdt.core.compiler.problem.discouragedReference=warning -org.eclipse.jdt.core.compiler.problem.emptyStatement=warning -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -<<<<<<< HEAD -======= -org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=warning -org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore -org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled -org.eclipse.jdt.core.compiler.problem.fieldHiding=warning -org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning -org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning -org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled -org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning -org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning -org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=warning -org.eclipse.jdt.core.compiler.problem.localVariableHiding=warning -org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning -org.eclipse.jdt.core.compiler.problem.missingDefaultCase=warning -org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=warning -org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled -org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=warning -org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=warning -org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled -org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning -org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=warning -org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error -org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=error -org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore -org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning -org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error -org.eclipse.jdt.core.compiler.problem.nullReference=warning -org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error -org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning -org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning -org.eclipse.jdt.core.compiler.problem.parameterAssignment=warning -org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=error -org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore -org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore -org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning -org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning -org.eclipse.jdt.core.compiler.problem.redundantNullCheck=warning -org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=warning -org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=warning -org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled -org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning -org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled -org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled -org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled -org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore -org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning -org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=disabled -org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning -org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning -org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore -org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryElse=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=warning -org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=warning -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled -org.eclipse.jdt.core.compiler.problem.unusedImport=warning -org.eclipse.jdt.core.compiler.problem.unusedLabel=warning -org.eclipse.jdt.core.compiler.problem.unusedLocal=warning -org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=warning -org.eclipse.jdt.core.compiler.problem.unusedParameter=warning -org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled -org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning -org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=warning -org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning -org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning ->>>>>>> ef72daa81c73f99e2156f5bfe8127591fc6358e9 -org.eclipse.jdt.core.compiler.source=1.7 diff --git a/.settings/org.eclipse.ltk.core.refactoring.prefs b/.settings/org.eclipse.ltk.core.refactoring.prefs new file mode 100644 index 000000000000..b196c64a3418 --- /dev/null +++ b/.settings/org.eclipse.ltk.core.refactoring.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index c924bef6be3c..b3f5def4f9fc 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -1,6 +1,8 @@ package com.google.gcloud; +import static com.google.common.base.MoreObjects.firstNonNull; + import com.google.api.client.extensions.appengine.http.UrlFetchTransport; import com.google.api.client.googleapis.compute.ComputeCredential; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; @@ -8,7 +10,6 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.jackson.JacksonFactory; -import com.google.common.base.MoreObjects; import java.io.IOException; import java.security.GeneralSecurityException; @@ -23,14 +24,14 @@ public abstract class ServiceOptions { private final AuthConfig authConfig; protected ServiceOptions(Builder builder) { - host = MoreObjects.firstNonNull(builder.host, DEFAULT_HOST); - httpTransport = MoreObjects.firstNonNull(builder.httpTransport, getDefaultHttpTransport()); - authConfig = MoreObjects.firstNonNull(builder.authConfig, getDefaultAuthConfig()); + host = firstNonNull(builder.host, DEFAULT_HOST); + httpTransport = firstNonNull(builder.httpTransport, getDefaultHttpTransport()); + authConfig = firstNonNull(builder.authConfig, getDefaultAuthConfig()); } private static HttpTransport getDefaultHttpTransport() { // Consider App Engine - if (System.getProperty("com.google.appengine.application.id") != null) { + if (getAppEngineAppId() != null) { try { return new UrlFetchTransport(); } catch (Exception ignore) { @@ -40,14 +41,15 @@ private static HttpTransport getDefaultHttpTransport() { // Consider Compute try { return getComputeCredential().getTransport(); - } catch (IOException | GeneralSecurityException e) { - return new NetHttpTransport(); + } catch (Exception e) { + // Maybe not on GCE } + return new NetHttpTransport(); } private static AuthConfig getDefaultAuthConfig() { // Consider App Engine - if (System.getProperty("com.google.appengine.application.id") != null) { + if (getAppEngineAppId() != null) { try { return AuthConfig.createForAppEngine(); } catch (Exception ignore) { @@ -63,9 +65,14 @@ private static AuthConfig getDefaultAuthConfig() { return cred; } }; - } catch (IOException | GeneralSecurityException e) { - return AuthConfig.createForAccount(null, null); + } catch (Exception ignore) { + // Maybe not on GCE } + return AuthConfig.createForAccount(null, null); + } + + protected static String getAppEngineAppId() { + return System.getProperty("com.google.appengine.application.id"); } private static ComputeCredential getComputeCredential() diff --git a/src/main/java/com/google/gcloud/datastore/CompleteKey.java b/src/main/java/com/google/gcloud/datastore/CompleteKey.java new file mode 100644 index 000000000000..1ee40971baee --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/CompleteKey.java @@ -0,0 +1,94 @@ +package com.google.gcloud.datastore; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.gcloud.datastore.Key.PathEntry; + +import java.util.List; + +/** + * A key that is guaranteed to be complete. + */ +public final class CompleteKey { + + private final Key key; + + public static class Builder { + + private final Key.Builder keyBuilder = new Key.Builder(); + + public Builder(String dataset) { + this(dataset, ""); + } + + public Builder(String dataset, String namespace) { + keyBuilder.setDataset(checkNotNull(dataset)); + keyBuilder.setNamespace(checkNotNull(namespace)); + } + + public Builder addChild(String kind, long id) { + keyBuilder.addChild(kind, id); + return this; + } + + public Builder addChild(String kind, String name) { + keyBuilder.addChild(kind, name); + return this; + } + + public CompleteKey build() { + return new CompleteKey(keyBuilder.build()); + } + } + + private CompleteKey(Key key) { + this.key = key; + } + + public String getDataset() { + return key.getDataset(); + } + + public String getNamespace() { + return key.getNamespace(); + } + + public List getPath() { + return key.getPath(); + } + + public String getKind() { + return key.getKind(); + } + + public Long getId() { + return key.getLeaf().getId(); + } + + public String getName() { + return key.getLeaf().getName(); + } + + public Object getNameOrId() { + PathEntry leaf = key.getLeaf(); + return leaf.hasId() ? leaf.getId() : leaf.getName(); + } + + @Override + public String toString() { + return key.toString(); + } + + @Override + public int hashCode() { + return key.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CompleteKey)) { + return false; + } + return key.equals(((CompleteKey) other).key); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 0923f42c5b19..c280bb30a42a 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -1,7 +1,20 @@ package com.google.gcloud.datastore; +import java.util.Map; + public interface DatastoreService { - String get(String key); + DatastoreServiceOptions getOptions(); + + CompleteKey put(Key key, Map values); + + Map get(CompleteKey key); + + void delete(CompleteKey... key); + + void allocateIds(Key... key); + + // query result item is a tuple of (key, value...) where values may be empty + //QueryResult runQuery(Query query); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 4793436fd6f0..5046c81969b9 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -8,10 +8,4 @@ final class DatastoreServiceImpl implements DatastoreService { this.options = options; } - @Override - public String get(String key) { - // TODO Auto-generated method stub - return null; - } - } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 5ece5b39e378..e9a3f9183c0e 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -1,21 +1,28 @@ package com.google.gcloud.datastore; +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import com.google.gcloud.ServiceOptions; import java.util.Set; +import java.util.regex.Pattern; public class DatastoreServiceOptions extends ServiceOptions { private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore"; private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email"; private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE); - + private static final Pattern PATTERN = Pattern.compile( + "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}\\:)?([a-z\\d][a-z\\d\\-]{0,99})"); private final String dataset; DatastoreServiceOptions(Builder builder) { super(builder); - dataset = builder.dataset; + dataset = firstNonNull(builder.dataset, getAppEngineAppId()); + checkArgument(dataset != null, "missing dataset"); } public static class Builder extends ServiceOptions.Builder { @@ -35,11 +42,22 @@ public DatastoreServiceOptions build() { } public Builder setDataset(String dataset) { - this.dataset = dataset; + this.dataset = validateDataset(dataset); return this; } } + static String validateDataset(String dataset) { + if (Strings.isNullOrEmpty(dataset)) { + throw new IllegalArgumentException("dataset can't be empty or null"); + } + if (!PATTERN.matcher(dataset).matches()) { + throw new IllegalArgumentException( + "dataset must match the following pattern: " + PATTERN.pattern()); + } + return dataset; + } + public String getDataset() { return dataset; } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java new file mode 100644 index 000000000000..cfb662569021 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -0,0 +1,183 @@ +package com.google.gcloud.datastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; + +import com.google.api.services.datastore.DatastoreV1.Key.PathElement; +import com.google.api.services.datastore.DatastoreV1.PartitionId; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +public final class Key { + + private final String dataset; + private final String namespace; + private final ImmutableList path; + + public static final class PathEntry { + + private final String kind; + private final Long id; + private final String name; + + PathEntry(String kind) { + this.kind = kind; + this.id = null; + this.name = null; + } + + PathEntry(String kind, long id) { + checkArgument(id != 0, "id must not be equal to zero"); + this.kind = kind; + this.id = id; + this.name = null; + } + + PathEntry(String kind, String name) { + checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); + checkArgument(name.length() <= 500, "name must not exceed 500 characters"); + this.kind = kind; + this.name = name; + this.id = null; + } + + public String getKind() { + return kind; + } + + public boolean hasId() { + return id != null; + } + + public long getId() { + return id == null ? 0 : id; + } + + public boolean hasName() { + return name != null; + } + + public String getName() { + return name == null ? "" : name; + } + } + + public static class Builder { + + private String dataset; + private String namespace; + private ImmutableList.Builder path = ImmutableList.builder(); + + public Builder addChild(String kind, long id) { + path.add(new PathEntry(kind, id)); + return this; + } + + public Builder addChild(String kind, String name) { + path.add(new PathEntry(kind, name)); + return this; + } + + public Builder addChild(String kind) { + path.add(new PathEntry(kind)); + return this; + } + + public Builder setDataset(String dataset) { + this.dataset = dataset == null ? null : validateDataset(dataset); + return this; + } + + public Builder setNamespace(String namespace) { + this.namespace = namespace; + return this; + } + + public Key build() { + return new Key(dataset, namespace, path.build()); + } + } + + private Key(String dataset, String namespace, ImmutableList path) { + checkState(!path.isEmpty(), "path must not be empty"); + this.dataset = dataset; + this.namespace = namespace; + this.path = path; + } + + public String getDataset() { + return dataset; + } + + public String getNamespace() { + return namespace; + } + + public List getPath() { + return path; + } + + public String getKind() { + return getLeaf().getKind(); + } + + @Override + public String toString() { + com.google.api.services.datastore.DatastoreV1.Value.Builder builder = + com.google.api.services.datastore.DatastoreV1.Value.newBuilder(); + addToPb(builder); + return builder.build().toString(); + } + + @Override + public int hashCode() { + return Objects.hash(dataset, namespace, path); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Key)) { + return false; + } + Key otherKey = (Key) other; + return Objects.equals(dataset, otherKey.dataset) + && Objects.equals(namespace, otherKey.namespace) + && Objects.equals(path, otherKey.path); + } + + PathEntry getLeaf() { + return path.get(path.size() - 1); + } + + void addToPb(com.google.api.services.datastore.DatastoreV1.Value.Builder builder) { + com.google.api.services.datastore.DatastoreV1.Key.Builder keyBuilder = + com.google.api.services.datastore.DatastoreV1.Key.newBuilder(); + com.google.api.services.datastore.DatastoreV1.PartitionId.Builder partitionBuilder = + PartitionId.newBuilder(); + if (dataset != null) { + partitionBuilder.setDatasetId(dataset); + } + if (namespace != null) { + partitionBuilder.setNamespace(namespace); + } + if (partitionBuilder.hasDatasetId() || partitionBuilder.hasNamespace()) { + keyBuilder.setPartitionId(partitionBuilder.build()); + } + for (PathEntry pathEntry : path) { + com.google.api.services.datastore.DatastoreV1.Key.PathElement.Builder pathElementBuilder = + PathElement.newBuilder(); + pathElementBuilder.setKind(pathEntry.kind); + if (pathEntry.id != null) { + pathElementBuilder.setId(pathEntry.id); + } else if (pathEntry.name != null) { + pathElementBuilder.setName(pathEntry.name); + } + keyBuilder.addPathElement(pathElementBuilder.build()); + } + builder.setKeyValue(keyBuilder.build()); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/KeyMapValue.java b/src/main/java/com/google/gcloud/datastore/KeyMapValue.java new file mode 100644 index 000000000000..6dcbea74fd97 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/KeyMapValue.java @@ -0,0 +1,24 @@ +package com.google.gcloud.datastore; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class KeyMapValue extends Value { + + private final Key key; + private final ImmutableMap values; + + KeyMapValue(boolean indexed) { + super(Type. + // TODO Auto-generated constructor stub + } + + public Key getKey() { + return key; + } + + public Map getValues() { + return values; + } +} diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java new file mode 100644 index 000000000000..3c9fe0f0621e --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/NullValue.java @@ -0,0 +1,31 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1.Value.Builder; + +public final class NullValue extends Value { + + static final Provider PROVIDER = new Provider() { + @Override + NullValue get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, + Integer meaning) { + return new NullValue(indexed, meaning); + } + }; + + public NullValue() { + this(true); + } + + public NullValue(boolean indexed) { + this(indexed, null); + } + + public NullValue(boolean indexed, Integer meaning) { + super(Type.NULL, indexed, meaning); + } + + @Override + protected void addValueToProto(Builder builder) { + // set nothing + } +} diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java new file mode 100644 index 000000000000..827fe7f43be3 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/StringValue.java @@ -0,0 +1,47 @@ +package com.google.gcloud.datastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.datastore.DatastoreV1.Value.Builder; + +// TODO: add javadoc, find the right place to describe that null should only +// be represented by NullValue (so nulls are not allowed here). +public final class StringValue extends Value { + + private final String content; + + static final Provider PROVIDER = new Provider() { + @Override + StringValue get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, + Integer meaning) { + return new StringValue(proto.getStringValue(), indexed, meaning); + } + }; + + public StringValue(String content) { + this(content, true); + } + + public StringValue(String content, boolean indexed) { + this(content, indexed, null); + } + + public StringValue(String content, boolean indexed, Integer meaning) { + super(Type.STRING, indexed, meaning); + this.content = checkNotNull(content); + // some validations: + if (indexed) { + checkArgument(content.length() <= 500, "Indexed string is limited to 500 characters"); + } + } + + public String getContent() { + return content; + } + + @Override + protected void addValueToProto(Builder builder) { + builder.setStringValue(content); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java new file mode 100644 index 000000000000..f600d9e4aef6 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -0,0 +1,129 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER; +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.api.services.datastore.DatastoreV1.Value.Builder; +import com.google.protobuf.Descriptors.FieldDescriptor; + +import java.util.Objects; + +public abstract class Value { + + abstract static class Provider { + + final V get(com.google.api.services.datastore.DatastoreV1.Value proto) { + return get(proto, proto.getIndexed(), proto.hasMeaning() ? proto.getMeaning() : null); + } + + abstract V get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, + Integer meaning); + } + + public enum Type { + NULL(NullValue.PROVIDER, 0), + STRING(StringValue.PROVIDER, STRING_VALUE_FIELD_NUMBER), + /* + TODO: implement + LONG(LongValue.class, INTEGER_VALUE_FIELD_NUMBER), + DOUBLE(DoubleValue.class, DOUBLE_VALUE_FIELD_NUMBER), + // TODO: make sure that getContent returns an immutable value or at least a copy + TIMESTAMP(TimestampValue.class, TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER), + BOOLEAN(BooleanValue.class, BOOLEAN_VALUE_FIELD_NUMBER), + BLOB(BlobValue.class, BLOB_VALUE_FIELD_NUMBER), + BLOB_KEY(BlobKeyValue.class, BLOB_KEY_VALUE_FIELD_NUMBER), + */ + // TODO: does not seem to be public... + // GEO_POINT(GeoPointValue.class, 8), + COMPLETE_KEY_VALUE(CompleteKeyValue.PROVIDER, KEY_VALUE_FIELD_NUMBER), + KEY_MAP_VALUE(KeyMapValue.class, ENTITY_VALUE_FIELD_NUMBER), + List_VALUE(ListValue.class, LIST_VALUE_FIELD_NUMBER); + + private final Provider provider; + private FieldDescriptor field; + + Type(Provider provider, int idx) { + this.provider = provider; + this.field = com.google.api.services.datastore.DatastoreV1.Value.getDescriptor() + .findFieldByNumber(idx); + } + + Provider getProvider() { + return provider; + } + + FieldDescriptor getDescriptor() { + return field; + } + } + + private final Type type; + private final boolean indexed; + private final Integer meaning; + + Value(Type type, boolean indexed, Integer meaning) { + this.type = type; + this.indexed = indexed; + this.meaning = meaning; + // some validations: + if (meaning != null) { + // TODO: consider supplying Ranges for allowed meaning and validating it here + // more specific validation could be done on the specific types themselves + // upon construction [e.g. integer with a meaning 13 should be in the range [0,100]] + if (indexed) { + checkArgument(meaning != 15 && meaning != 22, + "Indexed values should not have meaning with 15 or 22"); + } + } + } + + public final Type getType() { + return type; + } + + public final boolean isIndexed() { + return indexed; + } + + public final Integer getMeaning() { + return meaning; + } + + @Override + public int hashCode() { + return Objects.hash(type, indexed, meaning); + } + + @Override + public boolean equals(Object other) { + if (!getClass().isInstance(other)) { + return false; + } + + Value otherValue = (Value) other; + return Objects.equals(type, otherValue.type) + && Objects.equals(indexed, otherValue.indexed) + && Objects.equals(meaning, otherValue.meaning); + } + + final com.google.api.services.datastore.DatastoreV1.Value toProto() { + Builder builder = com.google.api.services.datastore.DatastoreV1.Value.newBuilder(); + builder.setIndexed(indexed); + if (meaning != null) { + builder.setMeaning(meaning); + } + addValueToProto(builder); + return builder.build(); + } + + static Value fromProto(com.google.api.services.datastore.DatastoreV1.Value proto) { + for (Type type : Type.values()) { + if (proto.hasField(type.getDescriptor())) { + return type.getProvider().get(proto); + } + } + return NullValue.PROVIDER.get(proto); + } + + protected abstract void addValueToProto(Builder builder); +} From 5a47498a0a87bc3a6c1cee97afd5e6a981d7e6f2 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 18 Nov 2014 16:36:01 -0800 Subject: [PATCH 012/732] Adding robusta config --- RobustaSettings.xml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 RobustaSettings.xml diff --git a/RobustaSettings.xml b/RobustaSettings.xml new file mode 100644 index 000000000000..e957a402fb2a --- /dev/null +++ b/RobustaSettings.xml @@ -0,0 +1,2 @@ + + From 9724180a570a4865fa16e599d9434bc9b74b1416 Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 19 Nov 2014 17:58:26 -0800 Subject: [PATCH 013/732] rename Value to Property. Also, work in progress for making Property with a Builder and clone/toString/toBuilder (to allow modification)... --- .classpath | 5 - .../com.google.appengine.eclipse.core.prefs | 2 + .settings/edu.umd.cs.findbugs.core.prefs | 137 ++++++++++++++++++ .../google/gcloud/datastore/CompleteKey.java | 10 ++ .../gcloud/datastore/DatastoreService.java | 4 +- .../datastore/DatastoreServiceImpl.java | 26 ++++ .../google/gcloud/datastore/KeyMapValue.java | 25 +++- .../google/gcloud/datastore/NullProperty.java | 42 ++++++ .../google/gcloud/datastore/NullValue.java | 31 ---- .../datastore/{Value.java => Property.java} | 86 +++++++---- .../{StringValue.java => StringProperty.java} | 16 +- 11 files changed, 305 insertions(+), 79 deletions(-) create mode 100644 .settings/com.google.appengine.eclipse.core.prefs create mode 100644 .settings/edu.umd.cs.findbugs.core.prefs create mode 100644 src/main/java/com/google/gcloud/datastore/NullProperty.java delete mode 100644 src/main/java/com/google/gcloud/datastore/NullValue.java rename src/main/java/com/google/gcloud/datastore/{Value.java => Property.java} (57%) rename src/main/java/com/google/gcloud/datastore/{StringValue.java => StringProperty.java} (63%) diff --git a/.classpath b/.classpath index f1ba61b8c0a1..70f23326e0d2 100644 --- a/.classpath +++ b/.classpath @@ -17,11 +17,6 @@ - - - - - diff --git a/.settings/com.google.appengine.eclipse.core.prefs b/.settings/com.google.appengine.eclipse.core.prefs new file mode 100644 index 000000000000..82c36afe4e35 --- /dev/null +++ b/.settings/com.google.appengine.eclipse.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +filesCopiedToWebInfLib= diff --git a/.settings/edu.umd.cs.findbugs.core.prefs b/.settings/edu.umd.cs.findbugs.core.prefs new file mode 100644 index 000000000000..f2b5b2f9de9c --- /dev/null +++ b/.settings/edu.umd.cs.findbugs.core.prefs @@ -0,0 +1,137 @@ +#FindBugs User Preferences +#Wed Nov 19 14:11:27 PST 2014 +cloud_id=edu.umd.cs.findbugs.cloud.doNothingCloud +detectorAppendingToAnObjectOutputStream=AppendingToAnObjectOutputStream|true +detectorAtomicityProblem=AtomicityProblem|true +detectorBadAppletConstructor=BadAppletConstructor|false +detectorBadResultSetAccess=BadResultSetAccess|true +detectorBadSyntaxForRegularExpression=BadSyntaxForRegularExpression|true +detectorBadUseOfReturnValue=BadUseOfReturnValue|true +detectorBadlyOverriddenAdapter=BadlyOverriddenAdapter|true +detectorBooleanReturnNull=BooleanReturnNull|true +detectorCallToUnsupportedMethod=CallToUnsupportedMethod|false +detectorCheckExpectedWarnings=CheckExpectedWarnings|false +detectorCheckImmutableAnnotation=CheckImmutableAnnotation|true +detectorCheckRelaxingNullnessAnnotation=CheckRelaxingNullnessAnnotation|true +detectorCheckTypeQualifiers=CheckTypeQualifiers|true +detectorCloneIdiom=CloneIdiom|true +detectorComparatorIdiom=ComparatorIdiom|true +detectorConfusedInheritance=ConfusedInheritance|true +detectorConfusionBetweenInheritedAndOuterMethod=ConfusionBetweenInheritedAndOuterMethod|true +detectorCrossSiteScripting=CrossSiteScripting|true +detectorDefaultEncodingDetector=DefaultEncodingDetector|true +detectorDoInsideDoPrivileged=DoInsideDoPrivileged|true +detectorDontCatchIllegalMonitorStateException=DontCatchIllegalMonitorStateException|true +detectorDontIgnoreResultOfPutIfAbsent=DontIgnoreResultOfPutIfAbsent|true +detectorDontUseEnum=DontUseEnum|true +detectorDroppedException=DroppedException|true +detectorDumbMethodInvocations=DumbMethodInvocations|true +detectorDumbMethods=DumbMethods|true +detectorDuplicateBranches=DuplicateBranches|true +detectorEmptyZipFileEntry=EmptyZipFileEntry|false +detectorEqualsOperandShouldHaveClassCompatibleWithThis=EqualsOperandShouldHaveClassCompatibleWithThis|true +detectorExplicitSerialization=ExplicitSerialization|true +detectorFinalizerNullsFields=FinalizerNullsFields|true +detectorFindBadCast2=FindBadCast2|true +detectorFindBadForLoop=FindBadForLoop|true +detectorFindCircularDependencies=FindCircularDependencies|false +detectorFindDeadLocalStores=FindDeadLocalStores|true +detectorFindDoubleCheck=FindDoubleCheck|true +detectorFindEmptySynchronizedBlock=FindEmptySynchronizedBlock|true +detectorFindFieldSelfAssignment=FindFieldSelfAssignment|true +detectorFindFinalizeInvocations=FindFinalizeInvocations|true +detectorFindFloatEquality=FindFloatEquality|true +detectorFindHEmismatch=FindHEmismatch|true +detectorFindInconsistentSync2=FindInconsistentSync2|true +detectorFindJSR166LockMonitorenter=FindJSR166LockMonitorenter|true +detectorFindLocalSelfAssignment2=FindLocalSelfAssignment2|true +detectorFindMaskedFields=FindMaskedFields|true +detectorFindMismatchedWaitOrNotify=FindMismatchedWaitOrNotify|true +detectorFindNakedNotify=FindNakedNotify|true +detectorFindNonShortCircuit=FindNonShortCircuit|true +detectorFindNullDeref=FindNullDeref|true +detectorFindNullDerefsInvolvingNonShortCircuitEvaluation=FindNullDerefsInvolvingNonShortCircuitEvaluation|true +detectorFindOpenStream=FindOpenStream|true +detectorFindPuzzlers=FindPuzzlers|true +detectorFindRefComparison=FindRefComparison|true +detectorFindReturnRef=FindReturnRef|true +detectorFindRoughConstants=FindRoughConstants|true +detectorFindRunInvocations=FindRunInvocations|true +detectorFindSelfComparison=FindSelfComparison|true +detectorFindSelfComparison2=FindSelfComparison2|true +detectorFindSleepWithLockHeld=FindSleepWithLockHeld|true +detectorFindSpinLoop=FindSpinLoop|true +detectorFindSqlInjection=FindSqlInjection|true +detectorFindTwoLockWait=FindTwoLockWait|true +detectorFindUncalledPrivateMethods=FindUncalledPrivateMethods|true +detectorFindUnconditionalWait=FindUnconditionalWait|true +detectorFindUninitializedGet=FindUninitializedGet|true +detectorFindUnrelatedTypesInGenericContainer=FindUnrelatedTypesInGenericContainer|true +detectorFindUnreleasedLock=FindUnreleasedLock|true +detectorFindUnsatisfiedObligation=FindUnsatisfiedObligation|true +detectorFindUnsyncGet=FindUnsyncGet|true +detectorFindUseOfNonSerializableValue=FindUseOfNonSerializableValue|true +detectorFindUselessControlFlow=FindUselessControlFlow|true +detectorFormatStringChecker=FormatStringChecker|true +detectorHugeSharedStringConstants=HugeSharedStringConstants|true +detectorIDivResultCastToDouble=IDivResultCastToDouble|true +detectorIncompatMask=IncompatMask|true +detectorInconsistentAnnotations=InconsistentAnnotations|true +detectorInefficientIndexOf=InefficientIndexOf|true +detectorInefficientMemberAccess=InefficientMemberAccess|false +detectorInefficientToArray=InefficientToArray|true +detectorInfiniteLoop=InfiniteLoop|true +detectorInfiniteRecursiveLoop=InfiniteRecursiveLoop|true +detectorInheritanceUnsafeGetResource=InheritanceUnsafeGetResource|true +detectorInitializationChain=InitializationChain|true +detectorInitializeNonnullFieldsInConstructor=InitializeNonnullFieldsInConstructor|true +detectorInstantiateStaticClass=InstantiateStaticClass|true +detectorIntCast2LongAsInstant=IntCast2LongAsInstant|true +detectorInvalidJUnitTest=InvalidJUnitTest|true +detectorIteratorIdioms=IteratorIdioms|true +detectorLazyInit=LazyInit|true +detectorLoadOfKnownNullValue=LoadOfKnownNullValue|true +detectorLostLoggerDueToWeakReference=LostLoggerDueToWeakReference|true +detectorMethodReturnCheck=MethodReturnCheck|true +detectorMultithreadedInstanceAccess=MultithreadedInstanceAccess|true +detectorMutableLock=MutableLock|true +detectorMutableStaticFields=MutableStaticFields|true +detectorNaming=Naming|true +detectorNoteUnconditionalParamDerefs=NoteUnconditionalParamDerefs|true +detectorNumberConstructor=NumberConstructor|true +detectorOptionalReturnNull=OptionalReturnNull|true +detectorOverridingEqualsNotSymmetrical=OverridingEqualsNotSymmetrical|true +detectorPreferZeroLengthArrays=PreferZeroLengthArrays|true +detectorPublicSemaphores=PublicSemaphores|false +detectorQuestionableBooleanAssignment=QuestionableBooleanAssignment|true +detectorReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass=ReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass|true +detectorReadReturnShouldBeChecked=ReadReturnShouldBeChecked|true +detectorRedundantInterfaces=RedundantInterfaces|true +detectorRepeatedConditionals=RepeatedConditionals|true +detectorRuntimeExceptionCapture=RuntimeExceptionCapture|true +detectorSerializableIdiom=SerializableIdiom|true +detectorStartInConstructor=StartInConstructor|true +detectorStaticCalendarDetector=StaticCalendarDetector|true +detectorStringConcatenation=StringConcatenation|true +detectorSuperfluousInstanceOf=SuperfluousInstanceOf|true +detectorSuspiciousThreadInterrupted=SuspiciousThreadInterrupted|true +detectorSwitchFallthrough=SwitchFallthrough|true +detectorSynchronizationOnSharedBuiltinConstant=SynchronizationOnSharedBuiltinConstant|true +detectorSynchronizeAndNullCheckField=SynchronizeAndNullCheckField|true +detectorSynchronizeOnClassLiteralNotGetClass=SynchronizeOnClassLiteralNotGetClass|true +detectorSynchronizingOnContentsOfFieldToProtectField=SynchronizingOnContentsOfFieldToProtectField|true +detectorURLProblems=URLProblems|true +detectorUncallableMethodOfAnonymousClass=UncallableMethodOfAnonymousClass|true +detectorUnnecessaryMath=UnnecessaryMath|true +detectorUnreadFields=UnreadFields|true +detectorUselessSubclassMethod=UselessSubclassMethod|false +detectorVarArgsProblems=VarArgsProblems|true +detectorVolatileUsage=VolatileUsage|true +detectorWaitInLoop=WaitInLoop|true +detectorWrongMapIterator=WrongMapIterator|true +detectorXMLFactoryBypass=XMLFactoryBypass|true +detector_threshold=2 +effort=default +filter_settings=Medium|BAD_PRACTICE,CORRECTNESS,EXPERIMENTAL,MT_CORRECTNESS,PERFORMANCE,STYLE|false|15 +filter_settings_neg=MALICIOUS_CODE,SECURITY,NOISE,I18N| +run_at_full_build=true diff --git a/src/main/java/com/google/gcloud/datastore/CompleteKey.java b/src/main/java/com/google/gcloud/datastore/CompleteKey.java index 1ee40971baee..fc66938f8920 100644 --- a/src/main/java/com/google/gcloud/datastore/CompleteKey.java +++ b/src/main/java/com/google/gcloud/datastore/CompleteKey.java @@ -74,6 +74,10 @@ public Object getNameOrId() { return leaf.hasId() ? leaf.getId() : leaf.getName(); } + public Key toKey() { + return key; + } + @Override public String toString() { return key.toString(); @@ -91,4 +95,10 @@ public boolean equals(Object other) { } return key.equals(((CompleteKey) other).key); } + + // TODO (here and in key - also consider for Properties): + // 1) cloneable + // 2) serializable (or externalizable) + // 3) toBuilder (and builder supports clear,...) + // 4) Builder accept builder } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index c280bb30a42a..44e517d10a07 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -6,9 +6,9 @@ public interface DatastoreService { DatastoreServiceOptions getOptions(); - CompleteKey put(Key key, Map values); + CompleteKey put(Key key, Map values); - Map get(CompleteKey key); + Map get(CompleteKey key); void delete(CompleteKey... key); diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 5046c81969b9..2c55473ec71c 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -1,5 +1,7 @@ package com.google.gcloud.datastore; +import java.util.Map; + final class DatastoreServiceImpl implements DatastoreService { private final DatastoreServiceOptions options; @@ -8,4 +10,28 @@ final class DatastoreServiceImpl implements DatastoreService { this.options = options; } + public DatastoreServiceOptions getOptions() { + // TODO Auto-generated method stub + return null; + } + + public CompleteKey put(Key key, Map values) { + // TODO Auto-generated method stub + return null; + } + + public Map get(CompleteKey key) { + // TODO Auto-generated method stub + return null; + } + + public void delete(CompleteKey... key) { + // TODO Auto-generated method stub + + } + + public void allocateIds(Key... key) { + // TODO Auto-generated method stub + + } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyMapValue.java b/src/main/java/com/google/gcloud/datastore/KeyMapValue.java index 6dcbea74fd97..7310bb49e341 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyMapValue.java +++ b/src/main/java/com/google/gcloud/datastore/KeyMapValue.java @@ -1,16 +1,28 @@ package com.google.gcloud.datastore; +import com.google.api.services.datastore.DatastoreV1.Value.Builder; import com.google.common.collect.ImmutableMap; import java.util.Map; -public class KeyMapValue extends Value { +public class KeyMapValue extends Property { private final Key key; - private final ImmutableMap values; + private final ImmutableMap values; + + static final Provider PROVIDER = new Provider() { + @Override + KeyMapValue get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, + Integer meaning) { + // TODO: implement + return new KeyMapValue(indexed); + } + }; KeyMapValue(boolean indexed) { - super(Type. + super(Type.KEY_MAP_VALUE, indexed, 0); + key = null; + values = null; // TODO Auto-generated constructor stub } @@ -18,7 +30,12 @@ public Key getKey() { return key; } - public Map getValues() { + public Map getValues() { return values; } + + @Override + protected void addValueToProto(Builder builder) { + // TODO Auto-generated method stub + } } diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullProperty.java new file mode 100644 index 000000000000..32c865d572fa --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/NullProperty.java @@ -0,0 +1,42 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1.Value; + +public final class NullProperty extends Property { + + static final Provider PROVIDER = new Provider() { + @Override + NullProperty get(Value proto, boolean indexed, Integer meaning) { + return new NullProperty(indexed, meaning); + } + }; + + public static class Builder extends Property.Builder { + + public Builder() { + super(Type.NULL); + } + + @Override + public NullProperty build() { + return new NullProperty(); + } + } + + public NullProperty() { + this(true); + } + + public NullProperty(boolean indexed) { + this(indexed, null); + } + + public NullProperty(boolean indexed, Integer meaning) { + super(Type.NULL, indexed, meaning); + } + + @Override + protected void addValueToProto(Builder builder) { + // set nothing + } +} diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java deleted file mode 100644 index 3c9fe0f0621e..000000000000 --- a/src/main/java/com/google/gcloud/datastore/NullValue.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.google.gcloud.datastore; - -import com.google.api.services.datastore.DatastoreV1.Value.Builder; - -public final class NullValue extends Value { - - static final Provider PROVIDER = new Provider() { - @Override - NullValue get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, - Integer meaning) { - return new NullValue(indexed, meaning); - } - }; - - public NullValue() { - this(true); - } - - public NullValue(boolean indexed) { - this(indexed, null); - } - - public NullValue(boolean indexed, Integer meaning) { - super(Type.NULL, indexed, meaning); - } - - @Override - protected void addValueToProto(Builder builder) { - // set nothing - } -} diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Property.java similarity index 57% rename from src/main/java/com/google/gcloud/datastore/Value.java rename to src/main/java/com/google/gcloud/datastore/Property.java index f600d9e4aef6..2f158cd872ae 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -1,30 +1,40 @@ package com.google.gcloud.datastore; +import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER; import static com.google.common.base.Preconditions.checkArgument; +import com.google.api.services.datastore.DatastoreV1.Value; import com.google.api.services.datastore.DatastoreV1.Value.Builder; import com.google.protobuf.Descriptors.FieldDescriptor; import java.util.Objects; -public abstract class Value { +public abstract class Property { - abstract static class Provider { + private final Type type; + private final boolean indexed; + private final Integer meaning; - final V get(com.google.api.services.datastore.DatastoreV1.Value proto) { + abstract static class Provider { + + final V get(Value proto) { return get(proto, proto.getIndexed(), proto.hasMeaning() ? proto.getMeaning() : null); } - abstract V get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, - Integer meaning); + abstract V get(Value proto, boolean indexed, Integer meaning); } public enum Type { - NULL(NullValue.PROVIDER, 0), - STRING(StringValue.PROVIDER, STRING_VALUE_FIELD_NUMBER), + + NULL(NullProperty.PROVIDER, 0), + STRING(StringProperty.PROVIDER, STRING_VALUE_FIELD_NUMBER), + // COMPLETE_KEY_VALUE(CompleteKeyValue.PROVIDER, KEY_VALUE_FIELD_NUMBER), + KEY_MAP_VALUE(KeyMapValue.PROVIDER, ENTITY_VALUE_FIELD_NUMBER); + // List_VALUE(ListValue.class, LIST_VALUE_FIELD_NUMBER); + /* - TODO: implement + TODO: Also implement LONG(LongValue.class, INTEGER_VALUE_FIELD_NUMBER), DOUBLE(DoubleValue.class, DOUBLE_VALUE_FIELD_NUMBER), // TODO: make sure that getContent returns an immutable value or at least a copy @@ -32,23 +42,18 @@ public enum Type { BOOLEAN(BooleanValue.class, BOOLEAN_VALUE_FIELD_NUMBER), BLOB(BlobValue.class, BLOB_VALUE_FIELD_NUMBER), BLOB_KEY(BlobKeyValue.class, BLOB_KEY_VALUE_FIELD_NUMBER), - */ - // TODO: does not seem to be public... - // GEO_POINT(GeoPointValue.class, 8), - COMPLETE_KEY_VALUE(CompleteKeyValue.PROVIDER, KEY_VALUE_FIELD_NUMBER), - KEY_MAP_VALUE(KeyMapValue.class, ENTITY_VALUE_FIELD_NUMBER), - List_VALUE(ListValue.class, LIST_VALUE_FIELD_NUMBER); - - private final Provider provider; + // GEO_POINT(GeoPointValue.class, 8) // Does not seem to be public yet... + */ + + private final Provider provider; private FieldDescriptor field; - Type(Provider provider, int idx) { + Type(Provider provider, int idx) { this.provider = provider; - this.field = com.google.api.services.datastore.DatastoreV1.Value.getDescriptor() - .findFieldByNumber(idx); + field = Value.getDescriptor().findFieldByNumber(idx); } - Provider getProvider() { + Provider getProvider() { return provider; } @@ -57,11 +62,28 @@ FieldDescriptor getDescriptor() { } } - private final Type type; - private final boolean indexed; - private final Integer meaning; + public static abstract class Builder

{ - Value(Type type, boolean indexed, Integer meaning) { + private final Type type; + private boolean indexed = true; + private Integer meaning; + + protected Builder(Type type) { + this.type = type; + } + + public void setIndexed(boolean indexed) { + this.indexed = indexed; + } + + public void setMeaning(Integer meaning) { + this.meaning = meaning; + } + + public abstract P build(); + } + + Property(Type type, boolean indexed, Integer meaning) { this.type = type; this.indexed = indexed; this.meaning = meaning; @@ -100,14 +122,14 @@ public boolean equals(Object other) { return false; } - Value otherValue = (Value) other; + Property otherValue = (Property) other; return Objects.equals(type, otherValue.type) && Objects.equals(indexed, otherValue.indexed) && Objects.equals(meaning, otherValue.meaning); } - final com.google.api.services.datastore.DatastoreV1.Value toProto() { - Builder builder = com.google.api.services.datastore.DatastoreV1.Value.newBuilder(); + final Value toProto() { + Builder builder = Value.newBuilder(); builder.setIndexed(indexed); if (meaning != null) { builder.setMeaning(meaning); @@ -116,13 +138,19 @@ final com.google.api.services.datastore.DatastoreV1.Value toProto() { return builder.build(); } - static Value fromProto(com.google.api.services.datastore.DatastoreV1.Value proto) { + // TODO: toString, clone?, serialize?, toBuilder, fromBuilder,... + + static Property fromProto(Value proto) { for (Type type : Type.values()) { if (proto.hasField(type.getDescriptor())) { return type.getProvider().get(proto); } } - return NullValue.PROVIDER.get(proto); + // change strategy to return RawProperty (package scope constructor) + // when no match instead of null. This could only be done + // when using the V1 API which added a NullValue type to distinct the cases + // and the use of oneof which generates an enum of all possible values. + return NullProperty.PROVIDER.get(proto); } protected abstract void addValueToProto(Builder builder); diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringProperty.java similarity index 63% rename from src/main/java/com/google/gcloud/datastore/StringValue.java rename to src/main/java/com/google/gcloud/datastore/StringProperty.java index 827fe7f43be3..4ad8b90408a0 100644 --- a/src/main/java/com/google/gcloud/datastore/StringValue.java +++ b/src/main/java/com/google/gcloud/datastore/StringProperty.java @@ -3,31 +3,31 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.services.datastore.DatastoreV1.Value; import com.google.api.services.datastore.DatastoreV1.Value.Builder; // TODO: add javadoc, find the right place to describe that null should only // be represented by NullValue (so nulls are not allowed here). -public final class StringValue extends Value { +public final class StringProperty extends Property { private final String content; - static final Provider PROVIDER = new Provider() { + static final Provider PROVIDER = new Provider() { @Override - StringValue get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, - Integer meaning) { - return new StringValue(proto.getStringValue(), indexed, meaning); + StringProperty get(Value proto, boolean indexed, Integer meaning) { + return new StringProperty(proto.getStringValue(), indexed, meaning); } }; - public StringValue(String content) { + public StringProperty(String content) { this(content, true); } - public StringValue(String content, boolean indexed) { + public StringProperty(String content, boolean indexed) { this(content, indexed, null); } - public StringValue(String content, boolean indexed, Integer meaning) { + public StringProperty(String content, boolean indexed, Integer meaning) { super(Type.STRING, indexed, meaning); this.content = checkNotNull(content); // some validations: From 36b70fdaddc131145bc96faeab0ad8958d9fcf2e Mon Sep 17 00:00:00 2001 From: ozarov Date: Wed, 19 Nov 2014 20:27:13 -0800 Subject: [PATCH 014/732] work in progress --- .../google/gcloud/datastore/NullProperty.java | 23 +++++++---- .../com/google/gcloud/datastore/Property.java | 41 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullProperty.java index 32c865d572fa..8ca9ed7efd80 100644 --- a/src/main/java/com/google/gcloud/datastore/NullProperty.java +++ b/src/main/java/com/google/gcloud/datastore/NullProperty.java @@ -2,7 +2,7 @@ import com.google.api.services.datastore.DatastoreV1.Value; -public final class NullProperty extends Property { +public final class NullProperty extends Property { static final Provider PROVIDER = new Provider() { @Override @@ -17,26 +17,31 @@ public Builder() { super(Type.NULL); } + public Builder(NullProperty property) { + super(property); + } + @Override public NullProperty build() { - return new NullProperty(); + return new NullProperty(this); } } public NullProperty() { - this(true); + this(new Builder()); } - public NullProperty(boolean indexed) { - this(indexed, null); + NullProperty(Builder builder) { + super(builder); } - public NullProperty(boolean indexed, Integer meaning) { - super(Type.NULL, indexed, meaning); + @Override + protected void addValueToProto(Value.Builder builder) { + // set nothing } @Override - protected void addValueToProto(Builder builder) { - // set nothing + public Property.Builder toBuilder() { + return new Builder(this); } } diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Property.java index 2f158cd872ae..8df283791098 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -5,24 +5,23 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.api.services.datastore.DatastoreV1.Value; -import com.google.api.services.datastore.DatastoreV1.Value.Builder; import com.google.protobuf.Descriptors.FieldDescriptor; import java.util.Objects; -public abstract class Property { +public abstract class Property

> { private final Type type; private final boolean indexed; private final Integer meaning; - abstract static class Provider { + abstract static class Provider

> { - final V get(Value proto) { + final P get(Value proto) { return get(proto, proto.getIndexed(), proto.hasMeaning() ? proto.getMeaning() : null); } - abstract V get(Value proto, boolean indexed, Integer meaning); + abstract P get(Value proto, boolean indexed, Integer meaning); } public enum Type { @@ -45,15 +44,15 @@ public enum Type { // GEO_POINT(GeoPointValue.class, 8) // Does not seem to be public yet... */ - private final Provider provider; + private final Provider> provider; private FieldDescriptor field; - Type(Provider provider, int idx) { + Type(Provider> provider, int idx) { this.provider = provider; field = Value.getDescriptor().findFieldByNumber(idx); } - Provider getProvider() { + Provider> getProvider() { return provider; } @@ -62,12 +61,18 @@ FieldDescriptor getDescriptor() { } } - public static abstract class Builder

{ + public abstract static class Builder

> { private final Type type; private boolean indexed = true; private Integer meaning; + protected Builder(Property

property) { + this.type = property.type; + this.indexed = property.indexed; + this.meaning = property.meaning; + } + protected Builder(Type type) { this.type = type; } @@ -83,10 +88,10 @@ public void setMeaning(Integer meaning) { public abstract P build(); } - Property(Type type, boolean indexed, Integer meaning) { - this.type = type; - this.indexed = indexed; - this.meaning = meaning; + Property(Builder

builder) { + this.type = builder.type; + this.indexed = builder.indexed; + this.meaning = builder.meaning; // some validations: if (meaning != null) { // TODO: consider supplying Ranges for allowed meaning and validating it here @@ -122,14 +127,14 @@ public boolean equals(Object other) { return false; } - Property otherValue = (Property) other; + Property otherValue = (Property) other; return Objects.equals(type, otherValue.type) && Objects.equals(indexed, otherValue.indexed) && Objects.equals(meaning, otherValue.meaning); } final Value toProto() { - Builder builder = Value.newBuilder(); + Value.Builder builder = Value.newBuilder(); builder.setIndexed(indexed); if (meaning != null) { builder.setMeaning(meaning); @@ -140,7 +145,7 @@ final Value toProto() { // TODO: toString, clone?, serialize?, toBuilder, fromBuilder,... - static Property fromProto(Value proto) { + static Property fromProto(Value proto) { for (Type type : Type.values()) { if (proto.hasField(type.getDescriptor())) { return type.getProvider().get(proto); @@ -153,5 +158,7 @@ static Property fromProto(Value proto) { return NullProperty.PROVIDER.get(proto); } - protected abstract void addValueToProto(Builder builder); + public abstract Builder

toBuilder(); + + protected abstract void addValueToProto(Value.Builder builder); } From b20b08414b611d0be6c156fb5cc454ec415e4035 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 20 Nov 2014 17:24:42 -0800 Subject: [PATCH 015/732] work in progress --- .../google/gcloud/datastore/KeyMapValue.java | 41 ---- .../gcloud/datastore/MapValueProperty.java | 63 ++++++ .../google/gcloud/datastore/NullProperty.java | 54 ++++-- .../com/google/gcloud/datastore/Property.java | 183 +++++++++++++----- .../gcloud/datastore/StringProperty.java | 73 ++++--- 5 files changed, 278 insertions(+), 136 deletions(-) delete mode 100644 src/main/java/com/google/gcloud/datastore/KeyMapValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/MapValueProperty.java diff --git a/src/main/java/com/google/gcloud/datastore/KeyMapValue.java b/src/main/java/com/google/gcloud/datastore/KeyMapValue.java deleted file mode 100644 index 7310bb49e341..000000000000 --- a/src/main/java/com/google/gcloud/datastore/KeyMapValue.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.google.gcloud.datastore; - -import com.google.api.services.datastore.DatastoreV1.Value.Builder; -import com.google.common.collect.ImmutableMap; - -import java.util.Map; - -public class KeyMapValue extends Property { - - private final Key key; - private final ImmutableMap values; - - static final Provider PROVIDER = new Provider() { - @Override - KeyMapValue get(com.google.api.services.datastore.DatastoreV1.Value proto, boolean indexed, - Integer meaning) { - // TODO: implement - return new KeyMapValue(indexed); - } - }; - - KeyMapValue(boolean indexed) { - super(Type.KEY_MAP_VALUE, indexed, 0); - key = null; - values = null; - // TODO Auto-generated constructor stub - } - - public Key getKey() { - return key; - } - - public Map getValues() { - return values; - } - - @Override - protected void addValueToProto(Builder builder) { - // TODO Auto-generated method stub - } -} diff --git a/src/main/java/com/google/gcloud/datastore/MapValueProperty.java b/src/main/java/com/google/gcloud/datastore/MapValueProperty.java new file mode 100644 index 000000000000..877cd33b6a36 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/MapValueProperty.java @@ -0,0 +1,63 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class MapValueProperty extends Property { + + private final Key key; + private final ImmutableMap values; + + static final Marshaller MARSHALLER = new Marshaller() { + @Override + MapValueProperty get(Value proto, boolean indexed, Integer meaning) { + // TODO: implement + return new MapValueProperty(indexed); + } + + @Override + public Builder newBuilder() { + // TODO Auto-generated method stub + return null; + } + + @Override + public MapValueProperty fromProto(Value proto) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Value toProto(MapValueProperty property) { + // TODO Auto-generated method stub + return null; + } + + @Override + public int getProtoFieldId() { + return ENTITY_VALUE_FIELD_NUMBER;; + } + }; + + MapValueProperty(boolean indexed) { + super(Type.KEY_MAP_VALUE, indexed, 0); + key = null; + values = null; + // TODO Auto-generated constructor stub + } + + public Key getKey() { + return key; + } + + public Map getValues() { + return values; + } + + @Override + protected void addValueToProto(Builder builder) { + // TODO Auto-generated method stub + } +} diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullProperty.java index 8ca9ed7efd80..570f85c22ad6 100644 --- a/src/main/java/com/google/gcloud/datastore/NullProperty.java +++ b/src/main/java/com/google/gcloud/datastore/NullProperty.java @@ -1,30 +1,56 @@ package com.google.gcloud.datastore; +import static com.google.common.base.Preconditions.checkArgument; + import com.google.api.services.datastore.DatastoreV1.Value; -public final class NullProperty extends Property { +public final class NullProperty extends Property { + + static final Marshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public Builder newBuilder() { + return new Builder(); + } + + @Override + public int getProtoFieldId() { + return 0; + } + + @Override + protected void set(Value from, Builder to) { + // nothing to set + } - static final Provider PROVIDER = new Provider() { @Override - NullProperty get(Value proto, boolean indexed, Integer meaning) { - return new NullProperty(indexed, meaning); + protected void set(NullProperty from, Value.Builder to) { + // nothing to set } }; - public static class Builder extends Property.Builder { + public static final class Builder extends Property.Builder { public Builder() { super(Type.NULL); } - public Builder(NullProperty property) { - super(property); - } - @Override public NullProperty build() { return new NullProperty(this); } + + @Override + public Void validate(Void value) { + checkArgument(value == null, "Only null values are allowed"); + return null; + } + + @Override + protected Builder self() { + return this; + } } public NullProperty() { @@ -34,14 +60,4 @@ public NullProperty() { NullProperty(Builder builder) { super(builder); } - - @Override - protected void addValueToProto(Value.Builder builder) { - // set nothing - } - - @Override - public Property.Builder toBuilder() { - return new Builder(this); - } } diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Property.java index 8df283791098..ddaecca27579 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -1,7 +1,5 @@ package com.google.gcloud.datastore; -import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; -import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER; import static com.google.common.base.Preconditions.checkArgument; import com.google.api.services.datastore.DatastoreV1.Value; @@ -9,27 +7,23 @@ import java.util.Objects; -public abstract class Property

> { +public abstract class Property, + B extends Property.Builder> { private final Type type; private final boolean indexed; private final Integer meaning; - - abstract static class Provider

> { - - final P get(Value proto) { - return get(proto, proto.getIndexed(), proto.hasMeaning() ? proto.getMeaning() : null); - } - - abstract P get(Value proto, boolean indexed, Integer meaning); - } + private final V value; public enum Type { - NULL(NullProperty.PROVIDER, 0), - STRING(StringProperty.PROVIDER, STRING_VALUE_FIELD_NUMBER), + NULL(NullProperty.MARSHALLER), + STRING(StringProperty.MARSHALLER); + // MapValue -> MapValueProperty + // ListValue -> ListValueProperty + // CompleteKey -> CompleteKeyProperty // COMPLETE_KEY_VALUE(CompleteKeyValue.PROVIDER, KEY_VALUE_FIELD_NUMBER), - KEY_MAP_VALUE(KeyMapValue.PROVIDER, ENTITY_VALUE_FIELD_NUMBER); + //KEY_MAP_VALUE(KeyMapValue.MARSHALLER); // List_VALUE(ListValue.class, LIST_VALUE_FIELD_NUMBER); /* @@ -44,16 +38,19 @@ public enum Type { // GEO_POINT(GeoPointValue.class, 8) // Does not seem to be public yet... */ - private final Provider> provider; + @SuppressWarnings("rawtypes") + private final Marshaller marshaller; private FieldDescriptor field; - Type(Provider> provider, int idx) { - this.provider = provider; - field = Value.getDescriptor().findFieldByNumber(idx); + , B extends Builder> + Type(Marshaller marshaller) { + this.marshaller = marshaller; + field = Value.getDescriptor().findFieldByNumber(marshaller.getProtoFieldId()); } - Provider> getProvider() { - return provider; + , B extends Builder> Marshaller + getMarshaller() { + return marshaller; } FieldDescriptor getDescriptor() { @@ -61,37 +58,109 @@ FieldDescriptor getDescriptor() { } } - public abstract static class Builder

> { + interface Marshaller, B extends Builder> { + + B newBuilder(); + + P fromProto(Value proto); + + Value toProto(P property); + + int getProtoFieldId(); + } + + abstract static class BaseMarshaller, B extends Builder> + implements Marshaller { + + @Override + public P fromProto(Value proto) { + B builder = newBuilder(); + builder.setIndexed(proto.getIndexed()); + if (proto.hasMeaning()) { + builder.setMeaning(proto.getMeaning()); + } + set(proto, builder); + return builder.build(); + } + + @Override + public final Value toProto(P property) { + Value.Builder builder = Value.newBuilder(); + builder.setIndexed(property.isIndexed()); + if (property.getMeaning() != null) { + builder.setMeaning(property.getMeaning()); + } + set(property, builder); + return builder.build(); + } + + protected abstract void set(Value from, B to); + + protected abstract void set(P from, Value.Builder to); + } + + public abstract static class Builder, B extends Builder> { private final Type type; private boolean indexed = true; private Integer meaning; - - protected Builder(Property

property) { - this.type = property.type; - this.indexed = property.indexed; - this.meaning = property.meaning; - } + private V value; protected Builder(Type type) { this.type = type; } - public void setIndexed(boolean indexed) { + public Type getType() { + return type; + } + + public B mergeFrom(P other) { + indexed = other.isIndexed(); + meaning = other.getMeaning(); + setValue(other.getValue()); + return self(); + } + + public boolean isIndexed() { + return indexed; + } + + public B setIndexed(boolean indexed) { this.indexed = indexed; + return self(); } - public void setMeaning(Integer meaning) { + public Integer getMeaning() { + return meaning; + } + + public B setMeaning(Integer meaning) { this.meaning = meaning; + return self(); + } + + public V getValue() { + return value; + } + + public B setValue(V value) { + this.value = validate(value); + return self(); } + protected V validate(V value) { + return value; + } + + protected abstract B self(); + public abstract P build(); } - Property(Builder

builder) { - this.type = builder.type; - this.indexed = builder.indexed; - this.meaning = builder.meaning; + Property(Builder builder) { + type = builder.getType(); + indexed = builder.isIndexed(); + meaning = builder.getMeaning(); // some validations: if (meaning != null) { // TODO: consider supplying Ranges for allowed meaning and validating it here @@ -102,6 +171,7 @@ public void setMeaning(Integer meaning) { "Indexed values should not have meaning with 15 or 22"); } } + value = builder.getValue(); } public final Type getType() { @@ -116,49 +186,56 @@ public final Integer getMeaning() { return meaning; } + public final V getValue() { + return value; + } + + @SuppressWarnings("unchecked") + public Builder toBuilder() { + @SuppressWarnings("rawtypes") + Builder builder = getType().getMarshaller().newBuilder(); + builder.mergeFrom(this); + return builder; + } + @Override public int hashCode() { - return Objects.hash(type, indexed, meaning); + return Objects.hash(getType(), isIndexed(), getMeaning()); } + @SuppressWarnings("unchecked") @Override public boolean equals(Object other) { if (!getClass().isInstance(other)) { return false; } - Property otherValue = (Property) other; - return Objects.equals(type, otherValue.type) - && Objects.equals(indexed, otherValue.indexed) - && Objects.equals(meaning, otherValue.meaning); + Property otherProperty = (Property) other; + return Objects.equals(type, otherProperty.getType()) + && Objects.equals(indexed, otherProperty.isIndexed()) + && Objects.equals(meaning, otherProperty.getMeaning()) + && Objects.equals(getValue(), otherProperty.getValue()); } + @SuppressWarnings({ "unchecked", "rawtypes" }) final Value toProto() { - Value.Builder builder = Value.newBuilder(); - builder.setIndexed(indexed); - if (meaning != null) { - builder.setMeaning(meaning); - } - addValueToProto(builder); - return builder.build(); + return ((Marshaller) getType().getMarshaller()).toProto(this); } // TODO: toString, clone?, serialize?, toBuilder, fromBuilder,... - static Property fromProto(Value proto) { + @SuppressWarnings("unchecked") + static , B extends Property.Builder> Property + fromProto(Value proto) { for (Type type : Type.values()) { if (proto.hasField(type.getDescriptor())) { - return type.getProvider().get(proto); + return (Property) type.getMarshaller().fromProto(proto); } } // change strategy to return RawProperty (package scope constructor) // when no match instead of null. This could only be done // when using the V1 API which added a NullValue type to distinct the cases // and the use of oneof which generates an enum of all possible values. - return NullProperty.PROVIDER.get(proto); + return (Property) NullProperty.MARSHALLER.fromProto(proto); } - - public abstract Builder

toBuilder(); - - protected abstract void addValueToProto(Value.Builder builder); } diff --git a/src/main/java/com/google/gcloud/datastore/StringProperty.java b/src/main/java/com/google/gcloud/datastore/StringProperty.java index 4ad8b90408a0..a012fdf6d2be 100644 --- a/src/main/java/com/google/gcloud/datastore/StringProperty.java +++ b/src/main/java/com/google/gcloud/datastore/StringProperty.java @@ -1,47 +1,74 @@ package com.google.gcloud.datastore; +import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.datastore.DatastoreV1.Value; -import com.google.api.services.datastore.DatastoreV1.Value.Builder; // TODO: add javadoc, find the right place to describe that null should only // be represented by NullValue (so nulls are not allowed here). -public final class StringProperty extends Property { +public final class StringProperty extends Property { - private final String content; + static final Marshaller MARSHALLER = + new BaseMarshaller() { - static final Provider PROVIDER = new Provider() { @Override - StringProperty get(Value proto, boolean indexed, Integer meaning) { - return new StringProperty(proto.getStringValue(), indexed, meaning); + public int getProtoFieldId() { + return STRING_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder() { + return new Builder(); + } + + @Override + protected void set(Value from, Builder to) { + to.setValue(from.getStringValue()); + } + + @Override + protected void set(StringProperty from, Value.Builder to) { + to.setStringValue(from.getValue()); } }; - public StringProperty(String content) { - this(content, true); - } + public static final class Builder extends Property.Builder { - public StringProperty(String content, boolean indexed) { - this(content, indexed, null); - } + Builder() { + this(""); + } + + public Builder(String value) { + super(Type.STRING); + setValue(value); + } - public StringProperty(String content, boolean indexed, Integer meaning) { - super(Type.STRING, indexed, meaning); - this.content = checkNotNull(content); - // some validations: - if (indexed) { - checkArgument(content.length() <= 500, "Indexed string is limited to 500 characters"); + @Override + public StringProperty build() { + if (isIndexed()) { + checkArgument(getValue().length() <= 500, "Indexed string is limited to 500 characters"); + } + return new StringProperty(this); + } + + @Override + public String validate(String value) { + return checkNotNull(value); + } + + @Override + protected Builder self() { + return this; } } - public String getContent() { - return content; + public StringProperty(String content) { + this(new Builder(content)); } - @Override - protected void addValueToProto(Builder builder) { - builder.setStringValue(content); + StringProperty(Builder builder) { + super(builder); } } From 688bce1a4dd9e08a3c37ea5ad73cfb03d9f56122 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 21 Nov 2014 16:02:28 -0800 Subject: [PATCH 016/732] work in progress --- .../java/com/google/gcloud/AuthConfig.java | 48 ++++++-- .../com/google/gcloud/ServiceOptions.java | 26 +--- .../java/com/google/gcloud/datastore/Key.java | 59 ++++++---- .../gcloud/datastore/MapValueProperty.java | 63 ---------- .../google/gcloud/datastore/NullProperty.java | 15 +-- .../com/google/gcloud/datastore/Property.java | 91 ++++++++------ .../gcloud/datastore/PropertyMapProperty.java | 111 ++++++++++++++++++ .../gcloud/datastore/StringProperty.java | 23 +--- 8 files changed, 253 insertions(+), 183 deletions(-) delete mode 100644 src/main/java/com/google/gcloud/datastore/MapValueProperty.java create mode 100644 src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java index 2bf34ae29ef8..d6321e5488d7 100644 --- a/src/main/java/com/google/gcloud/AuthConfig.java +++ b/src/main/java/com/google/gcloud/AuthConfig.java @@ -1,28 +1,21 @@ package com.google.gcloud; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.compute.ComputeCredential; import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.jackson.JacksonFactory; +import java.io.IOException; +import java.security.GeneralSecurityException; import java.security.PrivateKey; import java.util.Set; public abstract class AuthConfig { - protected abstract HttpRequestInitializer getHttpRequestInitializer( - HttpTransport transport, Set scopes); - - - public static AuthConfig createForAppEngine() { - return new AppEngineAuthConfig(); - } - - public static AuthConfig createForAccount(String account, PrivateKey privateKey) { - return new ServiceAccountAuthConfig(account, privateKey); - } - private static class AppEngineAuthConfig extends AuthConfig { @Override @@ -54,4 +47,35 @@ protected HttpRequestInitializer getHttpRequestInitializer( .build(); } } + + protected abstract HttpRequestInitializer getHttpRequestInitializer( + HttpTransport transport, Set scopes); + + + public static AuthConfig createForAppEngine() { + return new AppEngineAuthConfig(); + } + + public static AuthConfig createForComputeEngine() throws IOException, GeneralSecurityException { + final ComputeCredential cred = getComputeCredential(); + return new AuthConfig() { + @Override + protected HttpRequestInitializer getHttpRequestInitializer(HttpTransport ts, Set sc) { + return cred; + } + }; + } + + public static AuthConfig createForAccount(String account, PrivateKey privateKey) { + return new ServiceAccountAuthConfig(account, privateKey); + } + + static ComputeCredential getComputeCredential() throws IOException, GeneralSecurityException { + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + // Try to connect using Google Compute Engine service account credentials. + ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); + // Force token refresh to detect if we are running on Google Compute Engine. + credential.refreshToken(); + return credential; + } } diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index b3f5def4f9fc..974cfcddae1f 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -4,15 +4,9 @@ import static com.google.common.base.MoreObjects.firstNonNull; import com.google.api.client.extensions.appengine.http.UrlFetchTransport; -import com.google.api.client.googleapis.compute.ComputeCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.jackson.JacksonFactory; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.util.Set; public abstract class ServiceOptions { @@ -40,7 +34,7 @@ private static HttpTransport getDefaultHttpTransport() { } // Consider Compute try { - return getComputeCredential().getTransport(); + return AuthConfig.getComputeCredential().getTransport(); } catch (Exception e) { // Maybe not on GCE } @@ -58,13 +52,7 @@ private static AuthConfig getDefaultAuthConfig() { } // Consider Compute try { - final ComputeCredential cred = getComputeCredential(); - return new AuthConfig() { - @Override protected HttpRequestInitializer getHttpRequestInitializer( - HttpTransport transport, Set scopes) { - return cred; - } - }; + return AuthConfig.createForComputeEngine(); } catch (Exception ignore) { // Maybe not on GCE } @@ -75,16 +63,6 @@ protected static String getAppEngineAppId() { return System.getProperty("com.google.appengine.application.id"); } - private static ComputeCredential getComputeCredential() - throws IOException, GeneralSecurityException { - NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); - // Try to connect using Google Compute Engine service account credentials. - ComputeCredential credential = new ComputeCredential(transport, new JacksonFactory()); - // Force token refresh to detect if we are running on Google Compute Engine. - credential.refreshToken(); - return credential; - } - protected abstract static class Builder { private String host; diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index cfb662569021..0ad1f670c151 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -4,6 +4,7 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; +import com.google.api.services.datastore.DatastoreV1; import com.google.api.services.datastore.DatastoreV1.Key.PathElement; import com.google.api.services.datastore.DatastoreV1.PartitionId; import com.google.common.base.Strings; @@ -127,10 +128,7 @@ public String getKind() { @Override public String toString() { - com.google.api.services.datastore.DatastoreV1.Value.Builder builder = - com.google.api.services.datastore.DatastoreV1.Value.newBuilder(); - addToPb(builder); - return builder.build().toString(); + return toPb().toString(); } @Override @@ -153,31 +151,52 @@ PathEntry getLeaf() { return path.get(path.size() - 1); } - void addToPb(com.google.api.services.datastore.DatastoreV1.Value.Builder builder) { - com.google.api.services.datastore.DatastoreV1.Key.Builder keyBuilder = - com.google.api.services.datastore.DatastoreV1.Key.newBuilder(); - com.google.api.services.datastore.DatastoreV1.PartitionId.Builder partitionBuilder = - PartitionId.newBuilder(); + static Key fromPb(DatastoreV1.Key keyPb) { + Builder builder = new Builder(); + if (keyPb.hasPartitionId()) { + PartitionId partitionIdPb = keyPb.getPartitionId(); + if (partitionIdPb.hasDatasetId()) { + builder.setDataset(partitionIdPb.getDatasetId()); + } + if (partitionIdPb.hasNamespace()) { + builder.setNamespace(partitionIdPb.getNamespace()); + } + } + for (PathElement pathElementPb : keyPb.getPathElementList()) { + String kind = pathElementPb.getKind(); + if (pathElementPb.hasId()) { + builder.addChild(kind, pathElementPb.getId()); + } else if (pathElementPb.hasName()) { + builder.addChild(kind, pathElementPb.getName()); + } else { + builder.addChild(kind); + } + } + return builder.build(); + } + + DatastoreV1.Key toPb() { + DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); + PartitionId.Builder partitionIdPb = PartitionId.newBuilder(); if (dataset != null) { - partitionBuilder.setDatasetId(dataset); + partitionIdPb.setDatasetId(dataset); } if (namespace != null) { - partitionBuilder.setNamespace(namespace); + partitionIdPb.setNamespace(namespace); } - if (partitionBuilder.hasDatasetId() || partitionBuilder.hasNamespace()) { - keyBuilder.setPartitionId(partitionBuilder.build()); + if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { + keyPb.setPartitionId(partitionIdPb.build()); } for (PathEntry pathEntry : path) { - com.google.api.services.datastore.DatastoreV1.Key.PathElement.Builder pathElementBuilder = - PathElement.newBuilder(); - pathElementBuilder.setKind(pathEntry.kind); + PathElement.Builder pathElementPb = PathElement.newBuilder(); + pathElementPb.setKind(pathEntry.kind); if (pathEntry.id != null) { - pathElementBuilder.setId(pathEntry.id); + pathElementPb.setId(pathEntry.id); } else if (pathEntry.name != null) { - pathElementBuilder.setName(pathEntry.name); + pathElementPb.setName(pathEntry.name); } - keyBuilder.addPathElement(pathElementBuilder.build()); + keyPb.addPathElement(pathElementPb.build()); } - builder.setKeyValue(keyBuilder.build()); + return keyPb.build(); } } diff --git a/src/main/java/com/google/gcloud/datastore/MapValueProperty.java b/src/main/java/com/google/gcloud/datastore/MapValueProperty.java deleted file mode 100644 index 877cd33b6a36..000000000000 --- a/src/main/java/com/google/gcloud/datastore/MapValueProperty.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.google.gcloud.datastore; - -import com.google.api.services.datastore.DatastoreV1.Value; -import com.google.common.collect.ImmutableMap; - -import java.util.Map; - -public class MapValueProperty extends Property { - - private final Key key; - private final ImmutableMap values; - - static final Marshaller MARSHALLER = new Marshaller() { - @Override - MapValueProperty get(Value proto, boolean indexed, Integer meaning) { - // TODO: implement - return new MapValueProperty(indexed); - } - - @Override - public Builder newBuilder() { - // TODO Auto-generated method stub - return null; - } - - @Override - public MapValueProperty fromProto(Value proto) { - // TODO Auto-generated method stub - return null; - } - - @Override - public Value toProto(MapValueProperty property) { - // TODO Auto-generated method stub - return null; - } - - @Override - public int getProtoFieldId() { - return ENTITY_VALUE_FIELD_NUMBER;; - } - }; - - MapValueProperty(boolean indexed) { - super(Type.KEY_MAP_VALUE, indexed, 0); - key = null; - values = null; - // TODO Auto-generated constructor stub - } - - public Key getKey() { - return key; - } - - public Map getValues() { - return values; - } - - @Override - protected void addValueToProto(Builder builder) { - // TODO Auto-generated method stub - } -} diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullProperty.java index 570f85c22ad6..0a9073995c6a 100644 --- a/src/main/java/com/google/gcloud/datastore/NullProperty.java +++ b/src/main/java/com/google/gcloud/datastore/NullProperty.java @@ -10,7 +10,7 @@ public final class NullProperty extends Property() { @Override - public Builder newBuilder() { + public Builder newBuilder(Value from) { return new Builder(); } @@ -20,17 +20,12 @@ public int getProtoFieldId() { } @Override - protected void set(Value from, Builder to) { - // nothing to set - } - - @Override - protected void set(NullProperty from, Value.Builder to) { + protected void setValueField(NullProperty from, Value.Builder to) { // nothing to set } }; - public static final class Builder extends Property.Builder { + public static final class Builder extends Property.BaseBuilder { public Builder() { super(Type.NULL); @@ -42,9 +37,9 @@ public NullProperty build() { } @Override - public Void validate(Void value) { + public Builder setValue(Void value) { checkArgument(value == null, "Only null values are allowed"); - return null; + return this; } @Override diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Property.java index ddaecca27579..c4a884ad9c39 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -1,14 +1,15 @@ package com.google.gcloud.datastore; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.datastore.DatastoreV1.Value; import com.google.protobuf.Descriptors.FieldDescriptor; import java.util.Objects; -public abstract class Property, - B extends Property.Builder> { +public abstract class + Property, B extends Property.Builder> { private final Type type; private final boolean indexed; @@ -18,8 +19,8 @@ public abstract class Property, public enum Type { NULL(NullProperty.MARSHALLER), - STRING(StringProperty.MARSHALLER); - // MapValue -> MapValueProperty + STRING(StringProperty.MARSHALLER), + PROPERTY_MAP(PropertyMapProperty.MARSHALLER); // ListValue -> ListValueProperty // CompleteKey -> CompleteKeyProperty // COMPLETE_KEY_VALUE(CompleteKeyValue.PROVIDER, KEY_VALUE_FIELD_NUMBER), @@ -58,11 +59,30 @@ FieldDescriptor getDescriptor() { } } - interface Marshaller, B extends Builder> { + interface Builder, B extends Builder> { + + Type getType(); + + B mergeFrom(P other); + + boolean isIndexed(); - B newBuilder(); + B setIndexed(boolean indexed); - P fromProto(Value proto); + Integer getMeaning(); + + B setMeaning(Integer meaning); + + V getValue(); + + B setValue(V value); + + P build(); + } + + interface Marshaller, B extends Builder> { + + B fromProto(Value proto); Value toProto(P property); @@ -73,14 +93,13 @@ abstract static class BaseMarshaller, B extends B implements Marshaller { @Override - public P fromProto(Value proto) { - B builder = newBuilder(); + public final B fromProto(Value proto) { + B builder = newBuilder(proto); builder.setIndexed(proto.getIndexed()); if (proto.hasMeaning()) { builder.setMeaning(proto.getMeaning()); } - set(proto, builder); - return builder.build(); + return builder; } @Override @@ -90,30 +109,33 @@ public final Value toProto(P property) { if (property.getMeaning() != null) { builder.setMeaning(property.getMeaning()); } - set(property, builder); + setValueField(property, builder); return builder.build(); } - protected abstract void set(Value from, B to); + protected abstract B newBuilder(Value from); - protected abstract void set(P from, Value.Builder to); + protected abstract void setValueField(P from, Value.Builder to); } - public abstract static class Builder, B extends Builder> { + abstract static class BaseBuilder, B extends BaseBuilder> + implements Builder { private final Type type; private boolean indexed = true; private Integer meaning; private V value; - protected Builder(Type type) { + protected BaseBuilder(Type type) { this.type = type; } + @Override public Type getType() { return type; } + @Override public B mergeFrom(P other) { indexed = other.isIndexed(); meaning = other.getMeaning(); @@ -121,39 +143,42 @@ public B mergeFrom(P other) { return self(); } + @Override public boolean isIndexed() { return indexed; } + @Override public B setIndexed(boolean indexed) { this.indexed = indexed; return self(); } + @Override public Integer getMeaning() { return meaning; } + @Override public B setMeaning(Integer meaning) { this.meaning = meaning; return self(); } + @Override public V getValue() { return value; } + @Override public B setValue(V value) { - this.value = validate(value); + this.value = checkNotNull(value); return self(); } - protected V validate(V value) { - return value; - } - protected abstract B self(); + @Override public abstract P build(); } @@ -190,12 +215,9 @@ public final V getValue() { return value; } - @SuppressWarnings("unchecked") - public Builder toBuilder() { - @SuppressWarnings("rawtypes") - Builder builder = getType().getMarshaller().newBuilder(); - builder.mergeFrom(this); - return builder; + public final B toBuilder() { + Marshaller marshaller = getType().getMarshaller(); + return marshaller.fromProto(toPb()); } @Override @@ -217,25 +239,24 @@ public boolean equals(Object other) { && Objects.equals(getValue(), otherProperty.getValue()); } - @SuppressWarnings({ "unchecked", "rawtypes" }) - final Value toProto() { - return ((Marshaller) getType().getMarshaller()).toProto(this); + @SuppressWarnings("unchecked") + Value toPb() { + Marshaller marshaller = getType().getMarshaller(); + return marshaller.toProto((P) this); } - // TODO: toString, clone?, serialize?, toBuilder, fromBuilder,... - @SuppressWarnings("unchecked") static , B extends Property.Builder> Property - fromProto(Value proto) { + fromPb(Value proto) { for (Type type : Type.values()) { if (proto.hasField(type.getDescriptor())) { - return (Property) type.getMarshaller().fromProto(proto); + return (Property) type.getMarshaller().fromProto(proto).build(); } } // change strategy to return RawProperty (package scope constructor) // when no match instead of null. This could only be done // when using the V1 API which added a NullValue type to distinct the cases // and the use of oneof which generates an enum of all possible values. - return (Property) NullProperty.MARSHALLER.fromProto(proto); + return (Property) new NullProperty(); } } diff --git a/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java new file mode 100644 index 000000000000..a66aa57910be --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java @@ -0,0 +1,111 @@ +package com.google.gcloud.datastore; + +import static com.google.api.client.util.Preconditions.checkNotNull; +import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.api.services.datastore.DatastoreV1.Entity; +import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public final class PropertyMapProperty extends + Property>, PropertyMapProperty, PropertyMapProperty.Builder> { + + private final Key key; + + static final Marshaller>, PropertyMapProperty, Builder> MARSHALLER = + new BaseMarshaller>, PropertyMapProperty, Builder>() { + + @Override + public int getProtoFieldId() { + return ENTITY_VALUE_FIELD_NUMBER; + } + + @Override + protected Builder newBuilder(Value from) { + Builder builder = new Builder(); + Entity entityPb = from.getEntityValue(); + if (entityPb.hasKey()) { + builder.setKey(Key.fromPb(entityPb.getKey())); + } + for (DatastoreV1.Property propertyPb : entityPb.getPropertyList()) { + builder.addProperty(propertyPb.getName(), Property.fromPb(propertyPb.getValue())); + } + return builder; + } + + @Override + protected void setValueField(PropertyMapProperty from, DatastoreV1.Value.Builder to) { + Entity.Builder entityPb = Entity.newBuilder(); + entityPb.setKey(from.getKey().toPb()); + for (Map.Entry> entry : from.getValue().entrySet()) { + Property property = entry.getValue(); + DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); + propertyPb.setName(entry.getKey()); + propertyPb.setValue(property.toPb()); + } + } + }; + + public static final class Builder extends + Property.BaseBuilder>, PropertyMapProperty, Builder> { + + private Key key; + private ImmutableMap.Builder> mapBuilder = ImmutableMap.builder(); + + public Builder() { + super(Type.PROPERTY_MAP); + setIndexed(false); + } + + public Key getKey() { + return key; + } + + public Builder setKey(Key key) { + this.key = key; + return this; + } + + public Builder addProperty(String name, Property property) { + mapBuilder.put(name, property); + return this; + } + + @Override + public Builder setValue(Map> value) { + mapBuilder = ImmutableMap.>builder().putAll(checkNotNull(value)); + return this; + } + + @Override + public Map> getValue() { + return mapBuilder.build(); + } + + @Override + protected Builder self() { + return this; + } + + @Override + public PropertyMapProperty build() { + return new PropertyMapProperty(this); + } + } + + public PropertyMapProperty(Key key, Map> properties) { + this(new Builder().setValue(properties).setKey(key)); + } + + PropertyMapProperty(Builder builder) { + super(builder); + key = builder.key; + } + + public Key getKey() { + return key; + } +} diff --git a/src/main/java/com/google/gcloud/datastore/StringProperty.java b/src/main/java/com/google/gcloud/datastore/StringProperty.java index a012fdf6d2be..df83288969d1 100644 --- a/src/main/java/com/google/gcloud/datastore/StringProperty.java +++ b/src/main/java/com/google/gcloud/datastore/StringProperty.java @@ -2,7 +2,6 @@ import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.datastore.DatastoreV1.Value; @@ -19,26 +18,17 @@ public int getProtoFieldId() { } @Override - public Builder newBuilder() { - return new Builder(); + protected Builder newBuilder(Value from) { + return new Builder(from.getStringValue()); } @Override - protected void set(Value from, Builder to) { - to.setValue(from.getStringValue()); - } - - @Override - protected void set(StringProperty from, Value.Builder to) { + protected void setValueField(StringProperty from, Value.Builder to) { to.setStringValue(from.getValue()); } }; - public static final class Builder extends Property.Builder { - - Builder() { - this(""); - } + public static final class Builder extends Property.BaseBuilder { public Builder(String value) { super(Type.STRING); @@ -53,11 +43,6 @@ public StringProperty build() { return new StringProperty(this); } - @Override - public String validate(String value) { - return checkNotNull(value); - } - @Override protected Builder self() { return this; From 1157e324b8311d9e6f0b9ff94abff117b97d3ebc Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 24 Nov 2014 17:39:34 -0800 Subject: [PATCH 017/732] work-in-progress --- .../google/gcloud/datastore/CompleteKey.java | 104 ------ .../gcloud/datastore/DatastoreService.java | 72 +++- .../datastore/DatastoreServiceImpl.java | 25 +- .../gcloud/datastore/IncompleteKey.java | 313 ++++++++++++++++++ .../java/com/google/gcloud/datastore/Key.java | 195 +++-------- .../google/gcloud/datastore/KeyProperty.java | 55 +++ .../google/gcloud/datastore/NullProperty.java | 2 + .../com/google/gcloud/datastore/Property.java | 46 ++- .../datastore/PropertyListProperty.java | 87 +++++ .../gcloud/datastore/PropertyMapProperty.java | 16 +- .../gcloud/datastore/StringProperty.java | 4 +- 11 files changed, 607 insertions(+), 312 deletions(-) delete mode 100644 src/main/java/com/google/gcloud/datastore/CompleteKey.java create mode 100644 src/main/java/com/google/gcloud/datastore/IncompleteKey.java create mode 100644 src/main/java/com/google/gcloud/datastore/KeyProperty.java create mode 100644 src/main/java/com/google/gcloud/datastore/PropertyListProperty.java diff --git a/src/main/java/com/google/gcloud/datastore/CompleteKey.java b/src/main/java/com/google/gcloud/datastore/CompleteKey.java deleted file mode 100644 index fc66938f8920..000000000000 --- a/src/main/java/com/google/gcloud/datastore/CompleteKey.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.google.gcloud.datastore; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.gcloud.datastore.Key.PathEntry; - -import java.util.List; - -/** - * A key that is guaranteed to be complete. - */ -public final class CompleteKey { - - private final Key key; - - public static class Builder { - - private final Key.Builder keyBuilder = new Key.Builder(); - - public Builder(String dataset) { - this(dataset, ""); - } - - public Builder(String dataset, String namespace) { - keyBuilder.setDataset(checkNotNull(dataset)); - keyBuilder.setNamespace(checkNotNull(namespace)); - } - - public Builder addChild(String kind, long id) { - keyBuilder.addChild(kind, id); - return this; - } - - public Builder addChild(String kind, String name) { - keyBuilder.addChild(kind, name); - return this; - } - - public CompleteKey build() { - return new CompleteKey(keyBuilder.build()); - } - } - - private CompleteKey(Key key) { - this.key = key; - } - - public String getDataset() { - return key.getDataset(); - } - - public String getNamespace() { - return key.getNamespace(); - } - - public List getPath() { - return key.getPath(); - } - - public String getKind() { - return key.getKind(); - } - - public Long getId() { - return key.getLeaf().getId(); - } - - public String getName() { - return key.getLeaf().getName(); - } - - public Object getNameOrId() { - PathEntry leaf = key.getLeaf(); - return leaf.hasId() ? leaf.getId() : leaf.getName(); - } - - public Key toKey() { - return key; - } - - @Override - public String toString() { - return key.toString(); - } - - @Override - public int hashCode() { - return key.hashCode(); - } - - @Override - public boolean equals(Object other) { - if (!(other instanceof CompleteKey)) { - return false; - } - return key.equals(((CompleteKey) other).key); - } - - // TODO (here and in key - also consider for Properties): - // 1) cloneable - // 2) serializable (or externalizable) - // 3) toBuilder (and builder supports clear,...) - // 4) Builder accept builder -} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 44e517d10a07..b876b721df6b 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -1,20 +1,78 @@ package com.google.gcloud.datastore; +import java.util.Iterator; import java.util.Map; public interface DatastoreService { - DatastoreServiceOptions getOptions(); + interface DatastoreReader { + + Map> get(Key key); + + // return the result in the given order + Iterator>> get(Iterator key); + + // query result item is a tuple of (key, value...) where values may be empty + //QueryResult runQuery(Query query); + } + + + interface DatastoreWriter { + + Key add(IncompleteKey key, Map> values); + + void update(Key key , Map> values); + + Key put(IncompleteKey key, Map> values); + + void delete(Key key); + } + + + public interface Transaction extends DatastoreReader, DatastoreWriter { + + void commit(); - CompleteKey put(Key key, Map values); + void rollback(); + } + + public interface TransactionOptions { + + enum IsolationLevel { + SERIALIZABLE, SNAPSHOT; + } + + IsolationLevel getIsolationLevel(); + } + + public interface Batch extends DatastoreWriter { + + @Override + void add(Key key, Map> values); + + @Override + void update(Key key , Map> values); + + @Override + void put(Key key, Map> values); + + void submit(); + } + + public interface BatchOptions { + + } + + DatastoreServiceOptions getOptions(); - Map get(CompleteKey key); + Transaction newTransaction(TransactionOptions tsOptions); - void delete(CompleteKey... key); + Batch newBatch(); - void allocateIds(Key... key); + P - // query result item is a tuple of (key, value...) where values may be empty - //QueryResult runQuery(Query query); + Key allocateId(IncompleteKey key); + // results are returned in request order + Iterator allocateIds(Iterator key); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 2c55473ec71c..f324081edaad 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -1,6 +1,5 @@ package com.google.gcloud.datastore; -import java.util.Map; final class DatastoreServiceImpl implements DatastoreService { @@ -10,28 +9,8 @@ final class DatastoreServiceImpl implements DatastoreService { this.options = options; } + @Override public DatastoreServiceOptions getOptions() { - // TODO Auto-generated method stub - return null; - } - - public CompleteKey put(Key key, Map values) { - // TODO Auto-generated method stub - return null; - } - - public Map get(CompleteKey key) { - // TODO Auto-generated method stub - return null; - } - - public void delete(CompleteKey... key) { - // TODO Auto-generated method stub - - } - - public void allocateIds(Key... key) { - // TODO Auto-generated method stub - + return options; } } diff --git a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java new file mode 100644 index 000000000000..36855b2bd9ba --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java @@ -0,0 +1,313 @@ +package com.google.gcloud.datastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamException; +import java.io.Serializable; +import java.util.List; +import java.util.Objects; + +public class IncompleteKey implements Serializable { + + private static final long serialVersionUID = -75301206578793347L; + + private final transient String dataset; + private final transient String namespace; + private final transient ImmutableList path; + private transient DatastoreV1.Key temp; // only for deserialization + + public static final class PathEntry implements Serializable { + + private static final long serialVersionUID = -7968078857690784595L; + + private final transient String kind; + private final transient Long id; + private final transient String name; + private transient DatastoreV1.Key.PathElement tempPathElementPb; // only for deserialization + + private PathEntry(String kind) { + this(kind, null); + } + + public PathEntry(String kind, long id) { + this.kind = kind; + this.id = id; + this.name = null; + } + + public PathEntry(String kind, String name) { + this.kind = kind; + this.name = name; + this.id = null; + } + + public String getKind() { + return kind; + } + + public boolean hasId() { + return id != null; + } + + public long getId() { + return id == null ? 0 : id; + } + + public boolean hasName() { + return name != null; + } + + public String getName() { + return name == null ? "" : name; + } + + public Object getNameOrId() { + return id == null ? name : id; + } + + @Override + public String toString() { + return toPb().toString(); + } + + @Override + public int hashCode() { + return Objects.hash(kind, id, name); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof PathEntry)) { + return false; + } + PathEntry other = (PathEntry) obj; + return Objects.equals(kind, other.kind) + && Objects.equals(id, other.id) + && Objects.equals(name, other.name); + } + + static PathEntry fromPb(DatastoreV1.Key.PathElement pathElementPb) { + String kind = pathElementPb.getKind(); + if (pathElementPb.hasId()) { + return new PathEntry(kind, pathElementPb.getId()); + } else if (pathElementPb.hasName()) { + return new PathEntry(kind, pathElementPb.getName()); + } + return new PathEntry(kind); + } + + DatastoreV1.Key.PathElement toPb() { + DatastoreV1.Key.PathElement.Builder pathElementPb = DatastoreV1.Key.PathElement.newBuilder(); + pathElementPb.setKind(kind); + if (id != null) { + pathElementPb.setId(id); + } else if (name != null) { + pathElementPb.setName(name); + } + return pathElementPb.build(); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeObject(toPb().toByteArray()); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + byte[] bytes = (byte[]) in.readObject(); + tempPathElementPb = DatastoreV1.Key.PathElement.parseFrom(bytes); + } + + @SuppressWarnings("unused") + private Object readResolve() throws ObjectStreamException { + return fromPb(tempPathElementPb); + } + } + + public static class Builder { + + private String dataset; + private String namespace = DEFAULT_NAMESPACE; + private String kind; + private ImmutableList.Builder path = ImmutableList.builder(); + + private static final String DEFAULT_NAMESPACE = ""; + + public Builder(String dataset, String kind) { + this.dataset = validateDataset(dataset); + this.kind = validateKind(kind); + } + + public Builder(IncompleteKey key) { + dataset = key.dataset; + namespace = key.namespace; + path = ImmutableList.builder().addAll(key.getAncestorPath()); + kind = key.getKind(); + } + + public Builder addToPath(String kind, long id) { + checkArgument(id != 0, "id must not be equal to zero"); + path.add(new PathEntry(kind, id)); + return this; + } + + public Builder addToPath(String kind, String name) { + checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); + checkArgument(name.length() <= 500, "name must not exceed 500 characters"); + path.add(new PathEntry(kind, name)); + return this; + } + + public void addToPath(PathEntry pathEntry) { + path.add(pathEntry); + } + + public Builder setKind(String kind) { + this.kind = validateKind(kind); + return this; + } + + private String validateKind(String kind) { + checkArgument(Strings.isNullOrEmpty(kind), "kind must not be empty or null"); + checkArgument(kind.length() <= 500, "kind must not contain more than 500 characters"); + return kind; + } + + public Builder clearPath() { + path = ImmutableList.builder(); + return this; + } + + public Builder setDataset(String dataset) { + this.dataset = validateDataset(dataset); + return this; + } + + public Builder setNamespace(String namespace) { + this.namespace = checkNotNull(namespace); + return this; + } + + public IncompleteKey build() { + PathEntry leaf = new PathEntry(kind); + ImmutableList pathList = + ImmutableList.builder().addAll(path.build()).add(leaf).build(); + return new IncompleteKey(dataset, namespace, pathList); + } + } + + IncompleteKey(String dataset, String namespace, ImmutableList path) { + checkState(!path.isEmpty(), "path must not be empty"); + this.dataset = dataset; + this.namespace = namespace; + this.path = path; + } + + public String getDataset() { + return dataset; + } + + public String getNamespace() { + return namespace; + } + + protected List getFullPath() { + return path; + } + + public List getAncestorPath() { + return path.subList(0, path.size() - 1); + } + + public String getKind() { + return getLeaf().getKind(); + } + + @Override + public String toString() { + return toPb().toString(); + } + + @Override + public int hashCode() { + return Objects.hash(dataset, namespace, path); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof IncompleteKey)) { + return false; + } + IncompleteKey otherKey = (IncompleteKey) other; + return Objects.equals(dataset, otherKey.dataset) + && Objects.equals(namespace, otherKey.namespace) + && Objects.equals(path, otherKey.path); + } + + PathEntry getLeaf() { + return path.get(path.size() - 1); + } + + static IncompleteKey fromPb(DatastoreV1.Key keyPb) { + String dataset = null; + String namespace = null; + if (keyPb.hasPartitionId()) { + DatastoreV1.PartitionId partitionIdPb = keyPb.getPartitionId(); + if (partitionIdPb.hasDatasetId()) { + dataset = partitionIdPb.getDatasetId(); + } + if (partitionIdPb.hasNamespace()) { + namespace = partitionIdPb.getNamespace(); + } + } + ImmutableList.Builder pathBuilder = ImmutableList.builder(); + for (DatastoreV1.Key.PathElement pathElementPb : keyPb.getPathElementList()) { + pathBuilder.add(PathEntry.fromPb(pathElementPb)); + } + return new IncompleteKey(dataset, namespace, pathBuilder.build()); + } + + DatastoreV1.Key toPb() { + DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); + DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder(); + if (dataset != null) { + partitionIdPb.setDatasetId(dataset); + } + if (namespace != null) { + partitionIdPb.setNamespace(namespace); + } + if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { + keyPb.setPartitionId(partitionIdPb.build()); + } + for (PathEntry pathEntry : path) { + keyPb.addPathElement(pathEntry.toPb()); + } + return keyPb.build(); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeObject(toPb().toByteArray()); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + byte[] bytes = (byte[]) in.readObject(); + temp = DatastoreV1.Key.parseFrom(bytes); + } + + @SuppressWarnings("unused") + private Object readResolve() throws ObjectStreamException { + return fromPb(temp); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index 0ad1f670c151..6e886da4283b 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -1,202 +1,81 @@ package com.google.gcloud.datastore; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; +import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.datastore.DatastoreV1; -import com.google.api.services.datastore.DatastoreV1.Key.PathElement; -import com.google.api.services.datastore.DatastoreV1.PartitionId; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.Objects; +/** + * A key that is guaranteed to be complete. + */ +public final class Key extends IncompleteKey { -public final class Key { + private static final long serialVersionUID = 3160994559785491356L; - private final String dataset; - private final String namespace; - private final ImmutableList path; - - public static final class PathEntry { - - private final String kind; - private final Long id; - private final String name; - - PathEntry(String kind) { - this.kind = kind; - this.id = null; - this.name = null; - } - - PathEntry(String kind, long id) { - checkArgument(id != 0, "id must not be equal to zero"); - this.kind = kind; - this.id = id; - this.name = null; - } + public static class Builder { - PathEntry(String kind, String name) { - checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); - checkArgument(name.length() <= 500, "name must not exceed 500 characters"); - this.kind = kind; - this.name = name; - this.id = null; - } + private final IncompleteKey.Builder keyBuilder; + private String name; - public String getKind() { - return kind; + public Builder(String dataset, String kind) { + keyBuilder = new IncompleteKey.Builder(dataset, kind); } - public boolean hasId() { - return id != null; - } - - public long getId() { - return id == null ? 0 : id; - } + public Builder(Key key) { + keyBuilder = new IncompleteKey.Builder(key); - public boolean hasName() { - return name != null; + // set name or id } - public String getName() { - return name == null ? "" : name; - } - } - - public static class Builder { - - private String dataset; - private String namespace; - private ImmutableList.Builder path = ImmutableList.builder(); - - public Builder addChild(String kind, long id) { - path.add(new PathEntry(kind, id)); + public Builder setDataset(String dataset) { + keyBuilder.setDataset(checkNotNull(dataset)); return this; } - public Builder addChild(String kind, String name) { - path.add(new PathEntry(kind, name)); + public Builder setNamespace(String namespace) { + keyBuilder.setNamespace(checkNotNull(namespace)); return this; } - public Builder addChild(String kind) { - path.add(new PathEntry(kind)); + public Builder addToPath(String kind, long id) { + keyBuilder.addChild(kind, id); return this; } - public Builder setDataset(String dataset) { - this.dataset = dataset == null ? null : validateDataset(dataset); + public Builder addToPath(String kind, String name) { + keyBuilder.addChild(kind, name); return this; } - public Builder setNamespace(String namespace) { - this.namespace = namespace; + public Builder clearPath() { + keyBuilder.clearPath(); return this; } - public Key build() { - return new Key(dataset, namespace, path.build()); - } - } - - private Key(String dataset, String namespace, ImmutableList path) { - checkState(!path.isEmpty(), "path must not be empty"); - this.dataset = dataset; - this.namespace = namespace; - this.path = path; - } - - public String getDataset() { - return dataset; - } - - public String getNamespace() { - return namespace; - } - - public List getPath() { - return path; - } + public Key build(String name) { - public String getKind() { - return getLeaf().getKind(); + return new Key(keyBuilder.build()); + } } - @Override - public String toString() { - return toPb().toString(); + private Key(IncompleteKey key) { + super(key.getDataset(), key.getNamespace(), ImmutableList.copyOf(key.getFullPath())); } - @Override - public int hashCode() { - return Objects.hash(dataset, namespace, path); + public Long getId() { + return getLeaf().getId(); } - @Override - public boolean equals(Object other) { - if (!(other instanceof Key)) { - return false; - } - Key otherKey = (Key) other; - return Objects.equals(dataset, otherKey.dataset) - && Objects.equals(namespace, otherKey.namespace) - && Objects.equals(path, otherKey.path); + public String getName() { + return getLeaf().getName(); } - PathEntry getLeaf() { - return path.get(path.size() - 1); + public Object getNameOrId() { + PathEntry leaf = getLeaf(); + return leaf.hasId() ? leaf.getId() : leaf.getName(); } static Key fromPb(DatastoreV1.Key keyPb) { - Builder builder = new Builder(); - if (keyPb.hasPartitionId()) { - PartitionId partitionIdPb = keyPb.getPartitionId(); - if (partitionIdPb.hasDatasetId()) { - builder.setDataset(partitionIdPb.getDatasetId()); - } - if (partitionIdPb.hasNamespace()) { - builder.setNamespace(partitionIdPb.getNamespace()); - } - } - for (PathElement pathElementPb : keyPb.getPathElementList()) { - String kind = pathElementPb.getKind(); - if (pathElementPb.hasId()) { - builder.addChild(kind, pathElementPb.getId()); - } else if (pathElementPb.hasName()) { - builder.addChild(kind, pathElementPb.getName()); - } else { - builder.addChild(kind); - } - } - return builder.build(); - } - - DatastoreV1.Key toPb() { - DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); - PartitionId.Builder partitionIdPb = PartitionId.newBuilder(); - if (dataset != null) { - partitionIdPb.setDatasetId(dataset); - } - if (namespace != null) { - partitionIdPb.setNamespace(namespace); - } - if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { - keyPb.setPartitionId(partitionIdPb.build()); - } - for (PathEntry pathEntry : path) { - PathElement.Builder pathElementPb = PathElement.newBuilder(); - pathElementPb.setKind(pathEntry.kind); - if (pathEntry.id != null) { - pathElementPb.setId(pathEntry.id); - } else if (pathEntry.name != null) { - pathElementPb.setName(pathEntry.name); - } - keyPb.addPathElement(pathElementPb.build()); - } - return keyPb.build(); + return new Key(IncompleteKey.fromPb(keyPb)); } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyProperty.java b/src/main/java/com/google/gcloud/datastore/KeyProperty.java new file mode 100644 index 000000000000..e1a21df1c467 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/KeyProperty.java @@ -0,0 +1,55 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.KEY_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1.Value; + +public final class KeyProperty extends Property { + + private static final long serialVersionUID = -1318353707326704821L; + + static final Marshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return KEY_VALUE_FIELD_NUMBER; + } + + @Override + protected Builder newBuilder(Value from) { + return new Builder(Key.fromPb(from.getKeyValue())); + } + + @Override + protected void setValueField(KeyProperty from, Value.Builder to) { + to.setKeyValue(from.getValue().toPb()); + } + }; + + public static final class Builder extends Property.BaseBuilder { + + public Builder(Key value) { + super(Type.KEY); + setValue(value); + } + + @Override + public KeyProperty build() { + return new KeyProperty(this); + } + + @Override + protected Builder self() { + return this; + } + } + + public KeyProperty(Key key) { + this(new Builder(key)); + } + + KeyProperty(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullProperty.java index 0a9073995c6a..0ea395c391d7 100644 --- a/src/main/java/com/google/gcloud/datastore/NullProperty.java +++ b/src/main/java/com/google/gcloud/datastore/NullProperty.java @@ -6,6 +6,8 @@ public final class NullProperty extends Property { + private static final long serialVersionUID = 8497300779013002270L; + static final Marshaller MARSHALLER = new BaseMarshaller() { diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Property.java index c4a884ad9c39..11d8f182b6ec 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -6,26 +6,33 @@ import com.google.api.services.datastore.DatastoreV1.Value; import com.google.protobuf.Descriptors.FieldDescriptor; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamException; +import java.io.Serializable; import java.util.Objects; +// TODO: add javadoc, and mention that null should only be represented by NullValue. public abstract class - Property, B extends Property.Builder> { + Property, B extends Property.Builder> + implements Serializable { - private final Type type; - private final boolean indexed; - private final Integer meaning; - private final V value; + private static final long serialVersionUID = -1899638277588872742L; + + private transient final Type type; + private transient final boolean indexed; + private transient final Integer meaning; + private transient final V value; + private transient Value tempValuePb; // only for deserialization public enum Type { NULL(NullProperty.MARSHALLER), STRING(StringProperty.MARSHALLER), - PROPERTY_MAP(PropertyMapProperty.MARSHALLER); - // ListValue -> ListValueProperty - // CompleteKey -> CompleteKeyProperty - // COMPLETE_KEY_VALUE(CompleteKeyValue.PROVIDER, KEY_VALUE_FIELD_NUMBER), - //KEY_MAP_VALUE(KeyMapValue.MARSHALLER); - // List_VALUE(ListValue.class, LIST_VALUE_FIELD_NUMBER); + PROPERTY_MAP(PropertyMapProperty.MARSHALLER), + PROPERTY_LIST(PropertyListProperty.MARSHALLER), + KEY(KeyProperty.MARSHALLER); /* TODO: Also implement @@ -259,4 +266,21 @@ Value toPb() { // and the use of oneof which generates an enum of all possible values. return (Property) new NullProperty(); } + + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeObject(toPb().toByteArray()); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + byte[] bytes = (byte[]) in.readObject(); + tempValuePb = Value.parseFrom(bytes); + } + + @SuppressWarnings("unused") + private Object readResolve() throws ObjectStreamException { + return fromPb(tempValuePb); + } } diff --git a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java new file mode 100644 index 000000000000..72af5de1f7e8 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java @@ -0,0 +1,87 @@ +package com.google.gcloud.datastore; + +import static com.google.api.client.util.Preconditions.checkNotNull; +import static com.google.api.services.datastore.DatastoreV1.Value.LIST_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +public final class PropertyListProperty extends + Property>, PropertyListProperty, PropertyListProperty.Builder> { + + private static final long serialVersionUID = -5461475706792576395L; + + static final Marshaller>, PropertyListProperty, Builder> MARSHALLER = + new BaseMarshaller>, PropertyListProperty, Builder>() { + + @Override + public int getProtoFieldId() { + return LIST_VALUE_FIELD_NUMBER; + } + + @Override + protected Builder newBuilder(Value from) { + Builder builder = new Builder(); + for (Value valuePb : from.getListValueList()) { + builder.addProperty(Property.fromPb(valuePb)); + } + return builder; + } + + @Override + protected void setValueField(PropertyListProperty from, Value.Builder to) { + for (Property property : from.getValue()) { + to.addListValue(property.toPb()); + } + } + }; + + public static final class Builder extends + Property.BaseBuilder>, PropertyListProperty, Builder> { + + private ImmutableList.Builder> listBuilder = ImmutableList.builder(); + + public Builder() { + super(Type.PROPERTY_LIST); + setIndexed(false); + } + + public Builder addProperty(Property property) { + listBuilder.add(property); + return this; + } + + @Override + public Builder setValue(List> value) { + listBuilder = ImmutableList.>builder().addAll(checkNotNull(value)); + return this; + } + + @Override + public List> getValue() { + return listBuilder.build(); + } + + @Override + protected Builder self() { + return this; + } + + @Override + public PropertyListProperty build() { + Preconditions.checkState(!getValue().isEmpty(), "Property list could not be empty"); + return new PropertyListProperty(this); + } + } + + public PropertyListProperty(List> properties) { + this(new Builder().setValue(properties)); + } + + PropertyListProperty(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java index a66aa57910be..395deea72769 100644 --- a/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java +++ b/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java @@ -13,7 +13,9 @@ public final class PropertyMapProperty extends Property>, PropertyMapProperty, PropertyMapProperty.Builder> { - private final Key key; + private static final long serialVersionUID = -5461475706792576395L; + + private final IncompleteKey key; static final Marshaller>, PropertyMapProperty, Builder> MARSHALLER = new BaseMarshaller>, PropertyMapProperty, Builder>() { @@ -28,7 +30,7 @@ protected Builder newBuilder(Value from) { Builder builder = new Builder(); Entity entityPb = from.getEntityValue(); if (entityPb.hasKey()) { - builder.setKey(Key.fromPb(entityPb.getKey())); + builder.setKey(IncompleteKey.fromPb(entityPb.getKey())); } for (DatastoreV1.Property propertyPb : entityPb.getPropertyList()) { builder.addProperty(propertyPb.getName(), Property.fromPb(propertyPb.getValue())); @@ -52,7 +54,7 @@ protected void setValueField(PropertyMapProperty from, DatastoreV1.Value.Builder public static final class Builder extends Property.BaseBuilder>, PropertyMapProperty, Builder> { - private Key key; + private IncompleteKey key; private ImmutableMap.Builder> mapBuilder = ImmutableMap.builder(); public Builder() { @@ -60,11 +62,11 @@ public Builder() { setIndexed(false); } - public Key getKey() { + public IncompleteKey getKey() { return key; } - public Builder setKey(Key key) { + public Builder setKey(IncompleteKey key) { this.key = key; return this; } @@ -96,7 +98,7 @@ public PropertyMapProperty build() { } } - public PropertyMapProperty(Key key, Map> properties) { + public PropertyMapProperty(IncompleteKey key, Map> properties) { this(new Builder().setValue(properties).setKey(key)); } @@ -105,7 +107,7 @@ public PropertyMapProperty(Key key, Map> properties) { key = builder.key; } - public Key getKey() { + public IncompleteKey getKey() { return key; } } diff --git a/src/main/java/com/google/gcloud/datastore/StringProperty.java b/src/main/java/com/google/gcloud/datastore/StringProperty.java index df83288969d1..dd805cdb5297 100644 --- a/src/main/java/com/google/gcloud/datastore/StringProperty.java +++ b/src/main/java/com/google/gcloud/datastore/StringProperty.java @@ -5,10 +5,10 @@ import com.google.api.services.datastore.DatastoreV1.Value; -// TODO: add javadoc, find the right place to describe that null should only -// be represented by NullValue (so nulls are not allowed here). public final class StringProperty extends Property { + private static final long serialVersionUID = -3105699707394545523L; + static final Marshaller MARSHALLER = new BaseMarshaller() { From c5b5b75533e24c298692193e8f4a98d8102bdc86 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 25 Nov 2014 17:36:17 -0800 Subject: [PATCH 018/732] adding entity --- .../gcloud/datastore/DatastoreService.java | 3 +- .../gcloud/datastore/EmbeddedEntity.java | 151 ++++++++++++++++++ .../datastore/EmbeddedEntityProperty.java | 63 ++++++++ .../com/google/gcloud/datastore/Entity.java | 143 +++++++++++++++++ .../gcloud/datastore/IncompleteKey.java | 71 ++++---- .../java/com/google/gcloud/datastore/Key.java | 111 +++++++++++-- .../google/gcloud/datastore/KeyProperty.java | 11 +- .../google/gcloud/datastore/NullProperty.java | 9 +- .../com/google/gcloud/datastore/Property.java | 13 +- .../datastore/PropertyListProperty.java | 26 ++- .../gcloud/datastore/PropertyMapProperty.java | 113 ------------- .../gcloud/datastore/StringProperty.java | 11 +- 12 files changed, 541 insertions(+), 184 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java create mode 100644 src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java create mode 100644 src/main/java/com/google/gcloud/datastore/Entity.java delete mode 100644 src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index b876b721df6b..6e09050be0f0 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -16,6 +16,7 @@ interface DatastoreReader { //QueryResult runQuery(Query query); } + // TODO: remove all refrence of IncomplteKey (except allocate and use Entity instead of Map) interface DatastoreWriter { @@ -69,8 +70,6 @@ public interface BatchOptions { Batch newBatch(); - P - Key allocateId(IncompleteKey key); // results are returned in request order diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java b/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java new file mode 100644 index 000000000000..00d8c49c0973 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java @@ -0,0 +1,151 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.collect.ImmutableMap; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public final class EmbeddedEntity implements Serializable { + + private static final long serialVersionUID = 6492561268709192891L; + + private final transient IncompleteKey key; + private final transient ImmutableMap> properties; + private transient DatastoreV1.Entity tempEntityPb; // only for deserialization + + public static final class Builder { + + private IncompleteKey key; + private Map> properties = new HashMap<>(); + + public Builder() { + } + + public Builder(EmbeddedEntity entity) { + this.key = entity.key; + this.properties = new HashMap<>(entity.properties); + } + + public Builder(Entity entity) { + this.key = entity.getKey(); + this.properties = new HashMap<>(entity.getProperties()); + } + + public Builder setKey(IncompleteKey key) { + this.key = key; + return this; + } + + public Builder clearProperties() { + properties.clear(); + return this; + } + + public Builder removeProperty(String name) { + properties.remove(name); + return this; + } + + public Builder setProperty(String name, Property property) { + properties.put(name, property); + return this; + } + + public EmbeddedEntity build() { + return new EmbeddedEntity(this); + } + } + + private EmbeddedEntity(Builder builder) { + key = builder.key; + properties = ImmutableMap.copyOf(builder.properties); + } + + /** + * Returns the key or null if not provided. + */ + public IncompleteKey getKey() { + return key; + } + + public boolean hasProperty(String name) { + return properties.containsKey(name); + } + + public Property getProperty(String name) { + return properties.get(name); + } + + public Set getPropertyNames() { + return properties.keySet(); + } + + @Override + public String toString() { + return toPb().toString(); + } + + @Override + public int hashCode() { + return Objects.hash(key, properties); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof EmbeddedEntity)) { + return false; + } + EmbeddedEntity other = (EmbeddedEntity) obj; + return Objects.equals(key, other.key) + && Objects.equals(properties, other.properties); + } + + static EmbeddedEntity fromPb(DatastoreV1.Entity entityPb) { + Builder builder = new Builder(); + if (entityPb.hasKey()) { + builder.setKey(IncompleteKey.fromPb(entityPb.getKey())); + } + for (DatastoreV1.Property property : entityPb.getPropertyList()) { + builder.setProperty(property.getName(), Property.fromPb(property.getValue())); + } + return builder.build(); + } + + DatastoreV1.Entity toPb() { + DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); + if (key != null) { + entityPb.setKey(key.toPb()); + } + for (Map.Entry> entry : properties.entrySet()) { + DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); + propertyPb.setName(entry.getKey()); + propertyPb.setValue(entry.getValue().toPb()); + entityPb.addProperty(propertyPb.build()); + } + return entityPb.build(); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeObject(toPb().toByteArray()); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + byte[] bytes = (byte[]) in.readObject(); + tempEntityPb = DatastoreV1.Entity.parseFrom(bytes); + } + + @SuppressWarnings("unused") + private Object readResolve() throws ObjectStreamException { + return fromPb(tempEntityPb); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java b/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java new file mode 100644 index 000000000000..2f03ffbe2e78 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java @@ -0,0 +1,63 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public final class EmbeddedEntityProperty extends + Property { + + private static final long serialVersionUID = -5461475706792576395L; + + static final Marshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return ENTITY_VALUE_FIELD_NUMBER; + } + + @Override + protected Builder newBuilder(EmbeddedEntity value) { + return new Builder(value); + } + + @Override + protected EmbeddedEntity getValue(DatastoreV1.Value from) { + return EmbeddedEntity.fromPb(from.getEntityValue()); + } + + @Override + protected void setValue(EmbeddedEntityProperty from, DatastoreV1.Value.Builder to) { + to.setEntityValue(from.getValue().toPb()); + } + }; + + public static final class Builder extends + Property.BaseBuilder { + + public Builder(EmbeddedEntity entity) { + super(Type.EMBEDDED_ENTITY); + setIndexed(false); + setValue(entity); + } + + @Override + protected Builder self() { + return this; + } + + @Override + public EmbeddedEntityProperty build() { + return new EmbeddedEntityProperty(this); + } + } + + public EmbeddedEntityProperty(EmbeddedEntity entity) { + this(new Builder(entity)); + } + + EmbeddedEntityProperty(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java new file mode 100644 index 000000000000..6051b4a3a32e --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -0,0 +1,143 @@ +package com.google.gcloud.datastore; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public final class Entity implements Serializable { + + private static final long serialVersionUID = 432961565733066915L; + + private final transient Key key; + private final transient ImmutableMap> properties; + private transient DatastoreV1.Entity tempEntityPb; // only for deserialization + + public static final class Builder { + + private Key key; + private Map> properties; + + public Builder(Key key) { + this.key = checkNotNull(key); + this.properties = new HashMap<>(); + } + + public Builder(Entity entity) { + this.key = entity.key; + this.properties = new HashMap<>(entity.properties); + } + + public Builder clearProperties() { + properties.clear(); + return this; + } + + public Builder removeProperty(String name) { + properties.remove(name); + return this; + } + + public Builder setProperty(String name, Property property) { + properties.put(name, property); + return this; + } + + public Entity build() { + return new Entity(this); + } + } + + private Entity(Builder builder) { + key = builder.key; + properties = ImmutableMap.copyOf(builder.properties); + } + + public Key getKey() { + return key; + } + + public boolean hasProperty(String name) { + return properties.containsKey(name); + } + + public Property getProperty(String name) { + return properties.get(name); + } + + public Set getPropertyNames() { + return properties.keySet(); + } + + Map> getProperties() { + return properties; + } + + @Override + public String toString() { + return toPb().toString(); + } + + @Override + public int hashCode() { + return Objects.hash(key, properties); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Entity)) { + return false; + } + Entity other = (Entity) obj; + return Objects.equals(key, other.key) + && Objects.equals(properties, other.properties); + } + + static Entity fromPb(DatastoreV1.Entity entityPb) { + Preconditions.checkArgument(entityPb.hasKey()); + Builder builder = new Builder(Key.fromPb(entityPb.getKey())); + for (DatastoreV1.Property property : entityPb.getPropertyList()) { + builder.setProperty(property.getName(), Property.fromPb(property.getValue())); + } + return builder.build(); + } + + DatastoreV1.Entity toPb() { + DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); + entityPb.setKey(key.toPb()); + for (Map.Entry> entry : properties.entrySet()) { + DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); + propertyPb.setName(entry.getKey()); + propertyPb.setValue(entry.getValue().toPb()); + entityPb.addProperty(propertyPb.build()); + } + return entityPb.build(); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeObject(toPb().toByteArray()); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + byte[] bytes = (byte[]) in.readObject(); + tempEntityPb = DatastoreV1.Entity.parseFrom(bytes); + } + + @SuppressWarnings("unused") + private Object readResolve() throws ObjectStreamException { + return fromPb(tempEntityPb); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java index 36855b2bd9ba..859694d5b9cf 100644 --- a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java +++ b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java @@ -6,6 +6,7 @@ import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -14,6 +15,7 @@ import java.io.ObjectOutputStream; import java.io.ObjectStreamException; import java.io.Serializable; +import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -23,10 +25,10 @@ public class IncompleteKey implements Serializable { private final transient String dataset; private final transient String namespace; - private final transient ImmutableList path; - private transient DatastoreV1.Key temp; // only for deserialization + private final transient ImmutableList path; + private transient DatastoreV1.Key tempKeyPb; // only for deserialization - public static final class PathEntry implements Serializable { + public static final class PathElement implements Serializable { private static final long serialVersionUID = -7968078857690784595L; @@ -35,17 +37,17 @@ public static final class PathEntry implements Serializable { private final transient String name; private transient DatastoreV1.Key.PathElement tempPathElementPb; // only for deserialization - private PathEntry(String kind) { + private PathElement(String kind) { this(kind, null); } - public PathEntry(String kind, long id) { + public PathElement(String kind, long id) { this.kind = kind; this.id = id; this.name = null; } - public PathEntry(String kind, String name) { + public PathElement(String kind, String name) { this.kind = kind; this.name = name; this.id = null; @@ -87,23 +89,23 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof PathEntry)) { + if (!(obj instanceof PathElement)) { return false; } - PathEntry other = (PathEntry) obj; + PathElement other = (PathElement) obj; return Objects.equals(kind, other.kind) && Objects.equals(id, other.id) && Objects.equals(name, other.name); } - static PathEntry fromPb(DatastoreV1.Key.PathElement pathElementPb) { + static PathElement fromPb(DatastoreV1.Key.PathElement pathElementPb) { String kind = pathElementPb.getKind(); if (pathElementPb.hasId()) { - return new PathEntry(kind, pathElementPb.getId()); + return new PathElement(kind, pathElementPb.getId()); } else if (pathElementPb.hasName()) { - return new PathEntry(kind, pathElementPb.getName()); + return new PathElement(kind, pathElementPb.getName()); } - return new PathEntry(kind); + return new PathElement(kind); } DatastoreV1.Key.PathElement toPb() { @@ -134,14 +136,15 @@ private Object readResolve() throws ObjectStreamException { } } - public static class Builder { + public static final class Builder { private String dataset; private String namespace = DEFAULT_NAMESPACE; private String kind; - private ImmutableList.Builder path = ImmutableList.builder(); + private List path = new LinkedList<>(); private static final String DEFAULT_NAMESPACE = ""; + private static final int MAX_PATH = 100; public Builder(String dataset, String kind) { this.dataset = validateDataset(dataset); @@ -151,25 +154,25 @@ public Builder(String dataset, String kind) { public Builder(IncompleteKey key) { dataset = key.dataset; namespace = key.namespace; - path = ImmutableList.builder().addAll(key.getAncestorPath()); kind = key.getKind(); + path.addAll(key.getAncestorPath()); } public Builder addToPath(String kind, long id) { checkArgument(id != 0, "id must not be equal to zero"); - path.add(new PathEntry(kind, id)); - return this; + return addToPath(new PathElement(kind, id)); } public Builder addToPath(String kind, String name) { checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); checkArgument(name.length() <= 500, "name must not exceed 500 characters"); - path.add(new PathEntry(kind, name)); - return this; + return addToPath(new PathElement(kind, name)); } - public void addToPath(PathEntry pathEntry) { + public Builder addToPath(PathElement pathEntry) { + Preconditions.checkState(path.size() < MAX_PATH, "path can have at most 100 elements"); path.add(pathEntry); + return this; } public Builder setKind(String kind) { @@ -184,7 +187,7 @@ private String validateKind(String kind) { } public Builder clearPath() { - path = ImmutableList.builder(); + path.clear(); return this; } @@ -199,14 +202,14 @@ public Builder setNamespace(String namespace) { } public IncompleteKey build() { - PathEntry leaf = new PathEntry(kind); - ImmutableList pathList = - ImmutableList.builder().addAll(path.build()).add(leaf).build(); + PathElement leaf = new PathElement(kind); + ImmutableList pathList = + ImmutableList.builder().addAll(path).add(leaf).build(); return new IncompleteKey(dataset, namespace, pathList); } } - IncompleteKey(String dataset, String namespace, ImmutableList path) { + IncompleteKey(String dataset, String namespace, ImmutableList path) { checkState(!path.isEmpty(), "path must not be empty"); this.dataset = dataset; this.namespace = namespace; @@ -221,11 +224,7 @@ public String getNamespace() { return namespace; } - protected List getFullPath() { - return path; - } - - public List getAncestorPath() { + public List getAncestorPath() { return path.subList(0, path.size() - 1); } @@ -254,7 +253,7 @@ public boolean equals(Object other) { && Objects.equals(path, otherKey.path); } - PathEntry getLeaf() { + PathElement getLeaf() { return path.get(path.size() - 1); } @@ -270,9 +269,9 @@ static IncompleteKey fromPb(DatastoreV1.Key keyPb) { namespace = partitionIdPb.getNamespace(); } } - ImmutableList.Builder pathBuilder = ImmutableList.builder(); + ImmutableList.Builder pathBuilder = ImmutableList.builder(); for (DatastoreV1.Key.PathElement pathElementPb : keyPb.getPathElementList()) { - pathBuilder.add(PathEntry.fromPb(pathElementPb)); + pathBuilder.add(PathElement.fromPb(pathElementPb)); } return new IncompleteKey(dataset, namespace, pathBuilder.build()); } @@ -289,7 +288,7 @@ DatastoreV1.Key toPb() { if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { keyPb.setPartitionId(partitionIdPb.build()); } - for (PathEntry pathEntry : path) { + for (PathElement pathEntry : path) { keyPb.addPathElement(pathEntry.toPb()); } return keyPb.build(); @@ -303,11 +302,11 @@ private void writeObject(ObjectOutputStream out) throws IOException { private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); byte[] bytes = (byte[]) in.readObject(); - temp = DatastoreV1.Key.parseFrom(bytes); + tempKeyPb = DatastoreV1.Key.parseFrom(bytes); } @SuppressWarnings("unused") private Object readResolve() throws ObjectStreamException { - return fromPb(temp); + return fromPb(tempKeyPb); } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index 6e886da4283b..8e315a23a86c 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -1,9 +1,16 @@ package com.google.gcloud.datastore; import static com.google.common.base.Preconditions.checkNotNull; +import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.datastore.DatastoreV1; import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; /** * A key that is guaranteed to be complete. @@ -16,15 +23,26 @@ public static class Builder { private final IncompleteKey.Builder keyBuilder; private String name; + private Long id; + + public Builder(String dataset, String kind, String name) { + keyBuilder = new IncompleteKey.Builder(dataset, kind); + this.name = name; + } - public Builder(String dataset, String kind) { + public Builder(String dataset, String kind, long id) { keyBuilder = new IncompleteKey.Builder(dataset, kind); + this.id = id; } - public Builder(Key key) { + public Builder(IncompleteKey key, String name) { keyBuilder = new IncompleteKey.Builder(key); + this.name = name; + } - // set name or id + public Builder(IncompleteKey key, long id) { + keyBuilder = new IncompleteKey.Builder(key); + this.id = id; } public Builder setDataset(String dataset) { @@ -37,13 +55,25 @@ public Builder setNamespace(String namespace) { return this; } + public Builder setName(String name) { + this.name = name; + this.id = null; + return this; + } + + public Builder setId(long id) { + this.id = id; + this.name = null; + return this; + } + public Builder addToPath(String kind, long id) { - keyBuilder.addChild(kind, id); + keyBuilder.addToPath(kind, id); return this; } public Builder addToPath(String kind, String name) { - keyBuilder.addChild(kind, name); + keyBuilder.addToPath(kind, name); return this; } @@ -52,14 +82,18 @@ public Builder clearPath() { return this; } - public Key build(String name) { - - return new Key(keyBuilder.build()); + public Key build() { + IncompleteKey key = keyBuilder.build(); + return id == null ? new Key(key, name) : new Key(key, id); } } - private Key(IncompleteKey key) { - super(key.getDataset(), key.getNamespace(), ImmutableList.copyOf(key.getFullPath())); + private Key(IncompleteKey key, String name) { + super(key.getDataset(), key.getNamespace(), newPath(key, name)); + } + + private Key(IncompleteKey key, long id) { + super(key.getDataset(), key.getNamespace(), newPath(key, id)); } public Long getId() { @@ -71,11 +105,64 @@ public String getName() { } public Object getNameOrId() { - PathEntry leaf = getLeaf(); + PathElement leaf = getLeaf(); return leaf.hasId() ? leaf.getId() : leaf.getName(); } + public String toUrlSafe() { + try { + return URLEncoder.encode(toString(), UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unxpeced encoding exception", e); + } + } + + public static Key fromUrlSafe(String urlSafe) { + try { + String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name()); + DatastoreV1.Key keyPb = DatastoreV1.Key.parseFrom(ByteString.copyFromUtf8(utf8Str)); + return Key.fromPb(keyPb); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unxpeced decoding exception", e); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException("Could not parse key", e); + } + } + + /** + * Convert an {@code IncompleteKey} to a {@code Key} provided that the key has + * either name or id (complete). + + * @throws IllegalArgumentException if provided key is not complete. + */ + public static Key fromIncompleteKey(IncompleteKey key) { + if (key instanceof Key) { + return (Key) key; + } + PathElement leaf = key.getLeaf(); + if (leaf.hasId()) { + return new Key(key, leaf.getId()); + } else if (leaf.hasName()) { + return new Key(key, leaf.getName()); + } + throw new IllegalArgumentException("Key is missing name or id"); + } + static Key fromPb(DatastoreV1.Key keyPb) { - return new Key(IncompleteKey.fromPb(keyPb)); + return fromIncompleteKey(IncompleteKey.fromPb(keyPb)); + } + + private static ImmutableList newPath(IncompleteKey key, String name) { + ImmutableList.Builder path = ImmutableList.builder(); + path.addAll(key.getAncestorPath()); + path.add(new PathElement(key.getKind(), name)); + return path.build(); + } + + private static ImmutableList newPath(IncompleteKey key, long id) { + ImmutableList.Builder path = ImmutableList.builder(); + path.addAll(key.getAncestorPath()); + path.add(new PathElement(key.getKind(), id)); + return path.build(); } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyProperty.java b/src/main/java/com/google/gcloud/datastore/KeyProperty.java index e1a21df1c467..95e5eb9eef58 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyProperty.java +++ b/src/main/java/com/google/gcloud/datastore/KeyProperty.java @@ -17,12 +17,17 @@ public int getProtoFieldId() { } @Override - protected Builder newBuilder(Value from) { - return new Builder(Key.fromPb(from.getKeyValue())); + protected Builder newBuilder(Key key) { + return new Builder(key); } @Override - protected void setValueField(KeyProperty from, Value.Builder to) { + protected Key getValue(Value from) { + return Key.fromPb(from.getKeyValue()); + } + + @Override + protected void setValue(KeyProperty from, Value.Builder to) { to.setKeyValue(from.getValue().toPb()); } }; diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullProperty.java index 0ea395c391d7..ba1861bc4b2e 100644 --- a/src/main/java/com/google/gcloud/datastore/NullProperty.java +++ b/src/main/java/com/google/gcloud/datastore/NullProperty.java @@ -12,7 +12,7 @@ public final class NullProperty extends Property() { @Override - public Builder newBuilder(Value from) { + public Builder newBuilder(Void value) { return new Builder(); } @@ -22,7 +22,12 @@ public int getProtoFieldId() { } @Override - protected void setValueField(NullProperty from, Value.Builder to) { + protected Void getValue(Value from) { + return null; + } + + @Override + protected void setValue(NullProperty from, Value.Builder to) { // nothing to set } }; diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Property.java index 11d8f182b6ec..6632b8bf4678 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -30,7 +30,7 @@ public enum Type { NULL(NullProperty.MARSHALLER), STRING(StringProperty.MARSHALLER), - PROPERTY_MAP(PropertyMapProperty.MARSHALLER), + EMBEDDED_ENTITY(EmbeddedEntityProperty.MARSHALLER), PROPERTY_LIST(PropertyListProperty.MARSHALLER), KEY(KeyProperty.MARSHALLER); @@ -101,7 +101,7 @@ abstract static class BaseMarshaller, B extends B @Override public final B fromProto(Value proto) { - B builder = newBuilder(proto); + B builder = newBuilder(getValue(proto)); builder.setIndexed(proto.getIndexed()); if (proto.hasMeaning()) { builder.setMeaning(proto.getMeaning()); @@ -116,13 +116,16 @@ public final Value toProto(P property) { if (property.getMeaning() != null) { builder.setMeaning(property.getMeaning()); } - setValueField(property, builder); + setValue(property, builder); return builder.build(); } - protected abstract B newBuilder(Value from); + // Move to a Builder Factory + protected abstract B newBuilder(V value); - protected abstract void setValueField(P from, Value.Builder to); + protected abstract V getValue(Value from); + + protected abstract void setValue(P from, Value.Builder to); } abstract static class BaseBuilder, B extends BaseBuilder> diff --git a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java index 72af5de1f7e8..6bf9290e41e5 100644 --- a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java +++ b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java @@ -1,12 +1,12 @@ package com.google.gcloud.datastore; -import static com.google.api.client.util.Preconditions.checkNotNull; import static com.google.api.services.datastore.DatastoreV1.Value.LIST_VALUE_FIELD_NUMBER; import com.google.api.services.datastore.DatastoreV1.Value; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; public final class PropertyListProperty extends @@ -23,16 +23,21 @@ public int getProtoFieldId() { } @Override - protected Builder newBuilder(Value from) { - Builder builder = new Builder(); + protected Builder newBuilder(List> properties) { + return new Builder().setValue(properties); + } + + @Override + protected List> getValue(Value from) { + List> properties = new ArrayList<>(from.getListValueCount()); for (Value valuePb : from.getListValueList()) { - builder.addProperty(Property.fromPb(valuePb)); + properties.add(Property.fromPb(valuePb)); } - return builder; + return properties; } @Override - protected void setValueField(PropertyListProperty from, Value.Builder to) { + protected void setValue(PropertyListProperty from, Value.Builder to) { for (Property property : from.getValue()) { to.addListValue(property.toPb()); } @@ -50,13 +55,18 @@ public Builder() { } public Builder addProperty(Property property) { + Preconditions.checkArgument( + property.getType() != Type.PROPERTY_LIST, "Cannot contain another list"); listBuilder.add(property); return this; } @Override - public Builder setValue(List> value) { - listBuilder = ImmutableList.>builder().addAll(checkNotNull(value)); + public Builder setValue(List> properties) { + listBuilder = ImmutableList.>builder(); + for (Property property : properties) { + addProperty(property); + } return this; } diff --git a/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java deleted file mode 100644 index 395deea72769..000000000000 --- a/src/main/java/com/google/gcloud/datastore/PropertyMapProperty.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.google.gcloud.datastore; - -import static com.google.api.client.util.Preconditions.checkNotNull; -import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; - -import com.google.api.services.datastore.DatastoreV1; -import com.google.api.services.datastore.DatastoreV1.Entity; -import com.google.api.services.datastore.DatastoreV1.Value; -import com.google.common.collect.ImmutableMap; - -import java.util.Map; - -public final class PropertyMapProperty extends - Property>, PropertyMapProperty, PropertyMapProperty.Builder> { - - private static final long serialVersionUID = -5461475706792576395L; - - private final IncompleteKey key; - - static final Marshaller>, PropertyMapProperty, Builder> MARSHALLER = - new BaseMarshaller>, PropertyMapProperty, Builder>() { - - @Override - public int getProtoFieldId() { - return ENTITY_VALUE_FIELD_NUMBER; - } - - @Override - protected Builder newBuilder(Value from) { - Builder builder = new Builder(); - Entity entityPb = from.getEntityValue(); - if (entityPb.hasKey()) { - builder.setKey(IncompleteKey.fromPb(entityPb.getKey())); - } - for (DatastoreV1.Property propertyPb : entityPb.getPropertyList()) { - builder.addProperty(propertyPb.getName(), Property.fromPb(propertyPb.getValue())); - } - return builder; - } - - @Override - protected void setValueField(PropertyMapProperty from, DatastoreV1.Value.Builder to) { - Entity.Builder entityPb = Entity.newBuilder(); - entityPb.setKey(from.getKey().toPb()); - for (Map.Entry> entry : from.getValue().entrySet()) { - Property property = entry.getValue(); - DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); - propertyPb.setName(entry.getKey()); - propertyPb.setValue(property.toPb()); - } - } - }; - - public static final class Builder extends - Property.BaseBuilder>, PropertyMapProperty, Builder> { - - private IncompleteKey key; - private ImmutableMap.Builder> mapBuilder = ImmutableMap.builder(); - - public Builder() { - super(Type.PROPERTY_MAP); - setIndexed(false); - } - - public IncompleteKey getKey() { - return key; - } - - public Builder setKey(IncompleteKey key) { - this.key = key; - return this; - } - - public Builder addProperty(String name, Property property) { - mapBuilder.put(name, property); - return this; - } - - @Override - public Builder setValue(Map> value) { - mapBuilder = ImmutableMap.>builder().putAll(checkNotNull(value)); - return this; - } - - @Override - public Map> getValue() { - return mapBuilder.build(); - } - - @Override - protected Builder self() { - return this; - } - - @Override - public PropertyMapProperty build() { - return new PropertyMapProperty(this); - } - } - - public PropertyMapProperty(IncompleteKey key, Map> properties) { - this(new Builder().setValue(properties).setKey(key)); - } - - PropertyMapProperty(Builder builder) { - super(builder); - key = builder.key; - } - - public IncompleteKey getKey() { - return key; - } -} diff --git a/src/main/java/com/google/gcloud/datastore/StringProperty.java b/src/main/java/com/google/gcloud/datastore/StringProperty.java index dd805cdb5297..319d57144a50 100644 --- a/src/main/java/com/google/gcloud/datastore/StringProperty.java +++ b/src/main/java/com/google/gcloud/datastore/StringProperty.java @@ -18,12 +18,17 @@ public int getProtoFieldId() { } @Override - protected Builder newBuilder(Value from) { - return new Builder(from.getStringValue()); + protected Builder newBuilder(String value) { + return new Builder(value); } @Override - protected void setValueField(StringProperty from, Value.Builder to) { + protected String getValue(Value from) { + return from.getStringValue(); + } + + @Override + protected void setValue(StringProperty from, Value.Builder to) { to.setStringValue(from.getValue()); } }; From d0e243c2c9366a2927cb6938fa3109ddddf4cef0 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 26 Nov 2014 17:47:26 -0800 Subject: [PATCH 019/732] work in progress --- .classpath | 1 + pom.xml | 76 ++++++----- .../com/google/gcloud/ServiceOptions.java | 6 +- .../gcloud/datastore/DatastoreReader.java | 14 +++ .../gcloud/datastore/DatastoreService.java | 48 ++----- .../datastore/DatastoreServiceImpl.java | 62 +++++++++ .../datastore/DatastoreServiceOptions.java | 16 ++- .../gcloud/datastore/DatastoreWriter.java | 12 ++ .../gcloud/datastore/EmbeddedEntity.java | 11 +- .../datastore/EmbeddedEntityProperty.java | 15 +-- .../com/google/gcloud/datastore/Entity.java | 8 +- .../gcloud/datastore/IncompleteKey.java | 4 +- .../java/com/google/gcloud/datastore/Key.java | 7 ++ .../google/gcloud/datastore/KeyProperty.java | 13 +- .../google/gcloud/datastore/NullProperty.java | 11 +- .../com/google/gcloud/datastore/Property.java | 119 ++++++++++-------- .../datastore/PropertyListProperty.java | 37 +++--- .../gcloud/datastore/StringProperty.java | 17 +-- .../gcloud/datastore/SerializationTest.java | 97 ++++++++++++++ 19 files changed, 394 insertions(+), 180 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreReader.java create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreWriter.java create mode 100644 src/test/java/com/google/gcloud/datastore/SerializationTest.java diff --git a/.classpath b/.classpath index 70f23326e0d2..09fe957d54f4 100644 --- a/.classpath +++ b/.classpath @@ -23,6 +23,7 @@ + diff --git a/pom.xml b/pom.xml index 534629a90885..d82863493cd8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,32 +4,52 @@ git-demo 0.0.1-SNAPSHOT - - com.google.http-client - google-http-client - 1.18.0-rc - - - com.google.oauth-client - google-oauth-client - 1.18.0-rc - - - com.google.guava - guava - RELEASE - - - com.google.apis - - google-api-services-datastore-protobuf - - v1beta2-rev1-2.1.0 - - - com.google.api-client - google-api-client-appengine - 1.18.0-rc - + + com.google.http-client + google-http-client + 1.18.0-rc + + + com.google.oauth-client + google-oauth-client + 1.18.0-rc + + + com.google.guava + guava + RELEASE + + + com.google.apis + + google-api-services-datastore-protobuf + + v1beta2-rev1-2.1.0 + + + com.google.api-client + google-api-client-appengine + 1.18.0-rc + + + junit + junit + test + RELEASE + - \ No newline at end of file + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.5.1 + + 1.7 + 1.7 + UTF-8 + + + + + diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 974cfcddae1f..137b9856bf0d 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -79,17 +79,17 @@ protected Builder(ServiceOptions options) { protected abstract ServiceOptions build(); - public Builder setHost(String host) { + public Builder host(String host) { this.host = host; return this; } - public Builder setHttpTransport(HttpTransport httpTransport) { + public Builder httpTransport(HttpTransport httpTransport) { this.httpTransport = httpTransport; return this; } - public Builder setAuthConfig(AuthConfig authConfig) { + public Builder authConfig(AuthConfig authConfig) { this.authConfig = authConfig; return this; } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java new file mode 100644 index 000000000000..a1eb56aab46a --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java @@ -0,0 +1,14 @@ +package com.google.gcloud.datastore; + +import java.util.Iterator; + +public interface DatastoreReader { + + Entity get(Key key); + + // results are returned using request order + Iterator get(Key... key); + + // query result item is a tuple of (key, value...) where values may be empty + //QueryResult runQuery(Query query); +} \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 6e09050be0f0..ca4c55033322 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -1,32 +1,11 @@ package com.google.gcloud.datastore; import java.util.Iterator; -import java.util.Map; -public interface DatastoreService { +public interface DatastoreService extends DatastoreReader, DatastoreWriter { - interface DatastoreReader { + interface Query { - Map> get(Key key); - - // return the result in the given order - Iterator>> get(Iterator key); - - // query result item is a tuple of (key, value...) where values may be empty - //QueryResult runQuery(Query query); - } - - // TODO: remove all refrence of IncomplteKey (except allocate and use Entity instead of Map) - - interface DatastoreWriter { - - Key add(IncompleteKey key, Map> values); - - void update(Key key , Map> values); - - Key put(IncompleteKey key, Map> values); - - void delete(Key key); } @@ -43,35 +22,28 @@ enum IsolationLevel { SERIALIZABLE, SNAPSHOT; } - IsolationLevel getIsolationLevel(); - } - - public interface Batch extends DatastoreWriter { - @Override - void add(Key key, Map> values); + IsolationLevel getIsolationLevel(); - @Override - void update(Key key , Map> values); + boolean force(); + } - @Override - void put(Key key, Map> values); + public interface BatchWriter extends DatastoreWriter { void submit(); } public interface BatchOptions { - } DatastoreServiceOptions getOptions(); - Transaction newTransaction(TransactionOptions tsOptions); + Transaction newTransaction(TransactionOptions transactionOptions); - Batch newBatch(); + BatchWriter newBatchWriter(BatchOptions batchOptions); Key allocateId(IncompleteKey key); - // results are returned in request order - Iterator allocateIds(Iterator key); + // results are returned using request order + Iterator allocateIds(IncompleteKey... key); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index f324081edaad..cc5ed9fa3434 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -1,5 +1,7 @@ package com.google.gcloud.datastore; +import java.util.Iterator; + final class DatastoreServiceImpl implements DatastoreService { @@ -13,4 +15,64 @@ final class DatastoreServiceImpl implements DatastoreService { public DatastoreServiceOptions getOptions() { return options; } + + @Override + public Transaction newTransaction(TransactionOptions transactionOptions) { + // TODO Auto-generated method stub + return null; + } + + @Override + public BatchWriter newBatchWriter(BatchOptions batchOptions) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Key allocateId(IncompleteKey key) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Iterator allocateIds(IncompleteKey... key) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Entity get(Key key) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Iterator get(Key... key) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void add(Entity... entity) { + // TODO Auto-generated method stub + + } + + @Override + public void update(Entity... entity) { + // TODO Auto-generated method stub + + } + + @Override + public Key put(Entity... entity) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void delete(Key... key) { + // TODO Auto-generated method stub + + } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index e9a3f9183c0e..bf2a58978fc1 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -17,23 +17,28 @@ public class DatastoreServiceOptions extends ServiceOptions { private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE); private static final Pattern PATTERN = Pattern.compile( "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}\\:)?([a-z\\d][a-z\\d\\-]{0,99})"); + private final String dataset; + private final boolean force; DatastoreServiceOptions(Builder builder) { super(builder); dataset = firstNonNull(builder.dataset, getAppEngineAppId()); checkArgument(dataset != null, "missing dataset"); + force = builder.force; } public static class Builder extends ServiceOptions.Builder { private String dataset; + private boolean force = false; public Builder() {} public Builder(DatastoreServiceOptions options) { super(options); dataset = options.dataset; + force = options.force; } @Override @@ -41,10 +46,15 @@ public DatastoreServiceOptions build() { return new DatastoreServiceOptions(this); } - public Builder setDataset(String dataset) { + public Builder dataset(String dataset) { this.dataset = validateDataset(dataset); return this; } + + public Builder force(boolean force) { + this.force = force; + return this; + } } static String validateDataset(String dataset) { @@ -62,6 +72,10 @@ public String getDataset() { return dataset; } + public boolean getForce() { + return force; + } + @Override protected Set getScopes() { return SCOPES; diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java new file mode 100644 index 000000000000..005df9ff6800 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java @@ -0,0 +1,12 @@ +package com.google.gcloud.datastore; + +public interface DatastoreWriter { + + void add(Entity... entity); + + void update(Entity... entity); + + Key put(Entity... entity); + + void delete(Key... key); +} \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java b/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java index 00d8c49c0973..853d4edf5d55 100644 --- a/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java +++ b/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java @@ -1,7 +1,7 @@ package com.google.gcloud.datastore; import com.google.api.services.datastore.DatastoreV1; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; import java.io.IOException; import java.io.ObjectInputStream; @@ -18,7 +18,7 @@ public final class EmbeddedEntity implements Serializable { private static final long serialVersionUID = 6492561268709192891L; private final transient IncompleteKey key; - private final transient ImmutableMap> properties; + private final transient ImmutableSortedMap> properties; private transient DatastoreV1.Entity tempEntityPb; // only for deserialization public static final class Builder { @@ -66,7 +66,12 @@ public EmbeddedEntity build() { private EmbeddedEntity(Builder builder) { key = builder.key; - properties = ImmutableMap.copyOf(builder.properties); + properties = ImmutableSortedMap.copyOf(builder.properties); + } + + public EmbeddedEntity(Entity entity) { + key = entity.getKey(); + properties = entity.getProperties(); } /** diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java b/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java index 2f03ffbe2e78..d1d653ac9f07 100644 --- a/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java +++ b/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java @@ -9,7 +9,7 @@ public final class EmbeddedEntityProperty extends private static final long serialVersionUID = -5461475706792576395L; - static final Marshaller MARSHALLER = + static final BaseMarshaller MARSHALLER = new BaseMarshaller() { @Override @@ -18,7 +18,7 @@ public int getProtoFieldId() { } @Override - protected Builder newBuilder(EmbeddedEntity value) { + public Builder newBuilder(EmbeddedEntity value) { return new Builder(value); } @@ -29,7 +29,7 @@ protected EmbeddedEntity getValue(DatastoreV1.Value from) { @Override protected void setValue(EmbeddedEntityProperty from, DatastoreV1.Value.Builder to) { - to.setEntityValue(from.getValue().toPb()); + to.setEntityValue(from.get().toPb()); } }; @@ -38,13 +38,8 @@ public static final class Builder extends public Builder(EmbeddedEntity entity) { super(Type.EMBEDDED_ENTITY); - setIndexed(false); - setValue(entity); - } - - @Override - protected Builder self() { - return this; + indexed(false); + set(entity); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index 6051b4a3a32e..84f42b434ced 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -4,7 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; import java.io.IOException; import java.io.ObjectInputStream; @@ -21,7 +21,7 @@ public final class Entity implements Serializable { private static final long serialVersionUID = 432961565733066915L; private final transient Key key; - private final transient ImmutableMap> properties; + private final transient ImmutableSortedMap> properties; private transient DatastoreV1.Entity tempEntityPb; // only for deserialization public static final class Builder { @@ -61,7 +61,7 @@ public Entity build() { private Entity(Builder builder) { key = builder.key; - properties = ImmutableMap.copyOf(builder.properties); + properties = ImmutableSortedMap.copyOf(builder.properties); } public Key getKey() { @@ -80,7 +80,7 @@ public Set getPropertyNames() { return properties.keySet(); } - Map> getProperties() { + ImmutableSortedMap> getProperties() { return properties; } diff --git a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java index 859694d5b9cf..8e67a39b5f0d 100644 --- a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java +++ b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java @@ -181,7 +181,7 @@ public Builder setKind(String kind) { } private String validateKind(String kind) { - checkArgument(Strings.isNullOrEmpty(kind), "kind must not be empty or null"); + checkArgument(!Strings.isNullOrEmpty(kind), "kind must not be empty or null"); checkArgument(kind.length() <= 500, "kind must not contain more than 500 characters"); return kind; } @@ -306,7 +306,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE } @SuppressWarnings("unused") - private Object readResolve() throws ObjectStreamException { + protected Object readResolve() throws ObjectStreamException { return fromPb(tempKeyPb); } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index 8e315a23a86c..adad267b27b2 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -8,6 +8,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import java.io.ObjectStreamException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; @@ -152,6 +153,12 @@ static Key fromPb(DatastoreV1.Key keyPb) { return fromIncompleteKey(IncompleteKey.fromPb(keyPb)); } + @Override + @SuppressWarnings("unused") + protected Object readResolve() throws ObjectStreamException { + return fromIncompleteKey((IncompleteKey) super.readResolve()); + } + private static ImmutableList newPath(IncompleteKey key, String name) { ImmutableList.Builder path = ImmutableList.builder(); path.addAll(key.getAncestorPath()); diff --git a/src/main/java/com/google/gcloud/datastore/KeyProperty.java b/src/main/java/com/google/gcloud/datastore/KeyProperty.java index 95e5eb9eef58..fda5daa11dcb 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyProperty.java +++ b/src/main/java/com/google/gcloud/datastore/KeyProperty.java @@ -8,7 +8,7 @@ public final class KeyProperty extends Property MARSHALLER = + static final BaseMarshaller MARSHALLER = new BaseMarshaller() { @Override @@ -17,7 +17,7 @@ public int getProtoFieldId() { } @Override - protected Builder newBuilder(Key key) { + public Builder newBuilder(Key key) { return new Builder(key); } @@ -28,7 +28,7 @@ protected Key getValue(Value from) { @Override protected void setValue(KeyProperty from, Value.Builder to) { - to.setKeyValue(from.getValue().toPb()); + to.setKeyValue(from.get().toPb()); } }; @@ -36,18 +36,13 @@ public static final class Builder extends Property.BaseBuilder MARSHALLER = + static final BaseMarshaller MARSHALLER = new BaseMarshaller() { @Override @@ -44,19 +44,14 @@ public NullProperty build() { } @Override - public Builder setValue(Void value) { + public Builder set(Void value) { checkArgument(value == null, "Only null values are allowed"); return this; } - - @Override - protected Builder self() { - return this; - } } public NullProperty() { - this(new Builder()); + this(new Builder().indexed(true)); } NullProperty(Builder builder) { diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Property.java index 6632b8bf4678..50ee7ca79834 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -13,26 +13,26 @@ import java.io.Serializable; import java.util.Objects; -// TODO: add javadoc, and mention that null should only be represented by NullValue. +// TODO: add javadoc, and mention that null should only be represented by NullValue. public abstract class Property, B extends Property.Builder> implements Serializable { private static final long serialVersionUID = -1899638277588872742L; - private transient final Type type; - private transient final boolean indexed; - private transient final Integer meaning; - private transient final V value; + private final transient Type type; + private final transient boolean indexed; + private final transient Integer meaning; + private final transient V value; private transient Value tempValuePb; // only for deserialization public enum Type { - NULL(NullProperty.MARSHALLER), - STRING(StringProperty.MARSHALLER), - EMBEDDED_ENTITY(EmbeddedEntityProperty.MARSHALLER), - PROPERTY_LIST(PropertyListProperty.MARSHALLER), - KEY(KeyProperty.MARSHALLER); + NULL(NullProperty.MARSHALLER, NullProperty.MARSHALLER), + STRING(StringProperty.MARSHALLER, StringProperty.MARSHALLER), + EMBEDDED_ENTITY(EmbeddedEntityProperty.MARSHALLER, EmbeddedEntityProperty.MARSHALLER), + PROPERTY_LIST(PropertyListProperty.MARSHALLER, PropertyListProperty.MARSHALLER), + KEY(KeyProperty.MARSHALLER, KeyProperty.MARSHALLER); /* TODO: Also implement @@ -46,13 +46,14 @@ public enum Type { // GEO_POINT(GeoPointValue.class, 8) // Does not seem to be public yet... */ - @SuppressWarnings("rawtypes") - private final Marshaller marshaller; + @SuppressWarnings("rawtypes") private final BuilderFactory builderFactory; + @SuppressWarnings("rawtypes") private final Marshaller marshaller; private FieldDescriptor field; , B extends Builder> - Type(Marshaller marshaller) { + Type(Marshaller marshaller, BuilderFactory builderFactory) { this.marshaller = marshaller; + this.builderFactory = builderFactory; field = Value.getDescriptor().findFieldByNumber(marshaller.getProtoFieldId()); } @@ -61,28 +62,38 @@ public enum Type { return marshaller; } + , B extends Builder> BuilderFactory + getBuilderFactory() { + return builderFactory; + } + FieldDescriptor getDescriptor() { return field; } } + interface BuilderFactory, B extends Builder> { + + B newBuilder(V value); + } + interface Builder, B extends Builder> { Type getType(); B mergeFrom(P other); - boolean isIndexed(); + boolean getIndexed(); - B setIndexed(boolean indexed); + B indexed(boolean indexed); Integer getMeaning(); - B setMeaning(Integer meaning); + B meaning(Integer meaning); - V getValue(); + V get(); - B setValue(V value); + B set(V value); P build(); } @@ -97,14 +108,14 @@ interface Marshaller, B extends Builder> } abstract static class BaseMarshaller, B extends Builder> - implements Marshaller { + implements Marshaller, BuilderFactory { @Override public final B fromProto(Value proto) { B builder = newBuilder(getValue(proto)); - builder.setIndexed(proto.getIndexed()); + builder.indexed(proto.getIndexed()); if (proto.hasMeaning()) { - builder.setMeaning(proto.getMeaning()); + builder.meaning(proto.getMeaning()); } return builder; } @@ -112,7 +123,7 @@ public final B fromProto(Value proto) { @Override public final Value toProto(P property) { Value.Builder builder = Value.newBuilder(); - builder.setIndexed(property.isIndexed()); + builder.setIndexed(property.getIndexed()); if (property.getMeaning() != null) { builder.setMeaning(property.getMeaning()); } @@ -120,9 +131,6 @@ public final Value toProto(P property) { return builder.build(); } - // Move to a Builder Factory - protected abstract B newBuilder(V value); - protected abstract V getValue(Value from); protected abstract void setValue(P from, Value.Builder to); @@ -147,19 +155,19 @@ public Type getType() { @Override public B mergeFrom(P other) { - indexed = other.isIndexed(); + indexed = other.getIndexed(); meaning = other.getMeaning(); - setValue(other.getValue()); + set(other.get()); return self(); } @Override - public boolean isIndexed() { + public boolean getIndexed() { return indexed; } @Override - public B setIndexed(boolean indexed) { + public B indexed(boolean indexed) { this.indexed = indexed; return self(); } @@ -170,23 +178,26 @@ public Integer getMeaning() { } @Override - public B setMeaning(Integer meaning) { + public B meaning(Integer meaning) { this.meaning = meaning; return self(); } @Override - public V getValue() { + public V get() { return value; } @Override - public B setValue(V value) { + public B set(V value) { this.value = checkNotNull(value); return self(); } - protected abstract B self(); + @SuppressWarnings("unchecked") + private B self() { + return (B) this; + } @Override public abstract P build(); @@ -194,7 +205,7 @@ public B setValue(V value) { Property(Builder builder) { type = builder.getType(); - indexed = builder.isIndexed(); + indexed = builder.getIndexed(); meaning = builder.getMeaning(); // some validations: if (meaning != null) { @@ -206,14 +217,14 @@ public B setValue(V value) { "Indexed values should not have meaning with 15 or 22"); } } - value = builder.getValue(); + value = builder.get(); } public final Type getType() { return type; } - public final boolean isIndexed() { + public final boolean getIndexed() { return indexed; } @@ -221,18 +232,19 @@ public final Integer getMeaning() { return meaning; } - public final V getValue() { + public final V get() { return value; } public final B toBuilder() { - Marshaller marshaller = getType().getMarshaller(); - return marshaller.fromProto(toPb()); + BuilderFactory builderFactory = getType().getBuilderFactory(); + B builder = builderFactory.newBuilder(get()); + return builder.mergeFrom((P) this); } @Override public int hashCode() { - return Objects.hash(getType(), isIndexed(), getMeaning()); + return Objects.hash(type, indexed, meaning); } @SuppressWarnings("unchecked") @@ -244,9 +256,9 @@ public boolean equals(Object other) { Property otherProperty = (Property) other; return Objects.equals(type, otherProperty.getType()) - && Objects.equals(indexed, otherProperty.isIndexed()) + && Objects.equals(indexed, otherProperty.getIndexed()) && Objects.equals(meaning, otherProperty.getMeaning()) - && Objects.equals(getValue(), otherProperty.getValue()); + && Objects.equals(value, otherProperty.get()); } @SuppressWarnings("unchecked") @@ -255,22 +267,31 @@ Value toPb() { return marshaller.toProto((P) this); } - @SuppressWarnings("unchecked") + @SuppressWarnings({ "rawtypes", "unchecked" }) static , B extends Property.Builder> Property fromPb(Value proto) { + Marshaller marshaller = NullProperty.MARSHALLER; for (Type type : Type.values()) { - if (proto.hasField(type.getDescriptor())) { - return (Property) type.getMarshaller().fromProto(proto).build(); + FieldDescriptor descriptor = type.getDescriptor(); + if (descriptor != null) { + if (descriptor.isRepeated()) { + if (proto.getRepeatedFieldCount(descriptor) > 0) { + marshaller = type.getMarshaller(); + break; + } + } else if (proto.hasField(descriptor)) { + marshaller = type.getMarshaller(); + break; + } } } // change strategy to return RawProperty (package scope constructor) // when no match instead of null. This could only be done - // when using the V1 API which added a NullValue type to distinct the cases + // when using the V3 API which added a NullValue type to distinct the cases // and the use of oneof which generates an enum of all possible values. - return (Property) new NullProperty(); + return marshaller.fromProto(proto).build(); } - private void writeObject(ObjectOutputStream out) throws IOException { out.defaultWriteObject(); out.writeObject(toPb().toByteArray()); @@ -283,7 +304,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE } @SuppressWarnings("unused") - private Object readResolve() throws ObjectStreamException { + protected Object readResolve() throws ObjectStreamException { return fromPb(tempValuePb); } } diff --git a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java index 6bf9290e41e5..9bd63168a800 100644 --- a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java +++ b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java @@ -14,7 +14,7 @@ public final class PropertyListProperty extends private static final long serialVersionUID = -5461475706792576395L; - static final Marshaller>, PropertyListProperty, Builder> MARSHALLER = + static final BaseMarshaller>, PropertyListProperty, Builder> MARSHALLER = new BaseMarshaller>, PropertyListProperty, Builder>() { @Override @@ -23,8 +23,8 @@ public int getProtoFieldId() { } @Override - protected Builder newBuilder(List> properties) { - return new Builder().setValue(properties); + public Builder newBuilder(List> properties) { + return new Builder().set(properties); } @Override @@ -38,7 +38,7 @@ protected Builder newBuilder(List> properties) { @Override protected void setValue(PropertyListProperty from, Value.Builder to) { - for (Property property : from.getValue()) { + for (Property property : from.get()) { to.addListValue(property.toPb()); } } @@ -51,7 +51,7 @@ public static final class Builder extends public Builder() { super(Type.PROPERTY_LIST); - setIndexed(false); + indexed(false); } public Builder addProperty(Property property) { @@ -61,8 +61,18 @@ public Builder addProperty(Property property) { return this; } + public Builder addProperty(Property first, Property... other) { + addProperty(first); + for (Property property : other) { + Preconditions.checkArgument( + property.getType() != Type.PROPERTY_LIST, "Cannot contain another list"); + listBuilder.add(property); + } + return this; + } + @Override - public Builder setValue(List> properties) { + public Builder set(List> properties) { listBuilder = ImmutableList.>builder(); for (Property property : properties) { addProperty(property); @@ -71,24 +81,23 @@ public Builder setValue(List> properties) { } @Override - public List> getValue() { + public List> get() { return listBuilder.build(); } - @Override - protected Builder self() { - return this; - } - @Override public PropertyListProperty build() { - Preconditions.checkState(!getValue().isEmpty(), "Property list could not be empty"); + Preconditions.checkState(!get().isEmpty(), "Property list could not be empty"); return new PropertyListProperty(this); } } public PropertyListProperty(List> properties) { - this(new Builder().setValue(properties)); + this(new Builder().set(properties)); + } + + public PropertyListProperty(Property first, Property... other) { + this(new Builder().addProperty(first, other)); } PropertyListProperty(Builder builder) { diff --git a/src/main/java/com/google/gcloud/datastore/StringProperty.java b/src/main/java/com/google/gcloud/datastore/StringProperty.java index 319d57144a50..1b021cba0ef7 100644 --- a/src/main/java/com/google/gcloud/datastore/StringProperty.java +++ b/src/main/java/com/google/gcloud/datastore/StringProperty.java @@ -9,7 +9,7 @@ public final class StringProperty extends Property MARSHALLER = + static final BaseMarshaller MARSHALLER = new BaseMarshaller() { @Override @@ -18,7 +18,7 @@ public int getProtoFieldId() { } @Override - protected Builder newBuilder(String value) { + public Builder newBuilder(String value) { return new Builder(value); } @@ -29,7 +29,7 @@ protected String getValue(Value from) { @Override protected void setValue(StringProperty from, Value.Builder to) { - to.setStringValue(from.getValue()); + to.setStringValue(from.get()); } }; @@ -37,21 +37,16 @@ public static final class Builder extends Property.BaseBuilder typeToProperties = ImmutableMap.of( + Type.NULL, array(NULL_PROPERTY), + Type.KEY, array(KEY_PROPERTY), + Type.STRING, array(STRING_PROPERTY), + Type.EMBEDDED_ENTITY, array(EMBEDDED_ENTITY_PROPERTY1, EMBEDDED_ENTITY_PROPERTY2, + EMBEDDED_ENTITY_PROPERTY3), + Type.PROPERTY_LIST, array(PROPERTY_LIST_PROPERTY)); + + private static T[] array(T... values) { + return values; + } + + @Test + public void testProperties() throws Exception { + for (Type type : Type.values()) { + Property[] properties = typeToProperties.get(type); + for (Property property : properties) { + Property copy = serialiazeAndDeserialize(property); + assertEquals(property, copy); + assertEquals(property.get(), copy.get()); + } + } + } + + @Test + public void testTypes() throws Exception { + Object[] types = new Object[] { KEY1, KEY2, INCOMPLETE_KEY, ENTITY1, ENTITY2, EMBEDDED_ENTITY1, + EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; + for (Object obj : types) { + Object copy = serialiazeAndDeserialize(obj); + assertEquals(obj, copy); + } + } + + private T serialiazeAndDeserialize(T obj) throws Exception { + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + ObjectOutputStream out = new ObjectOutputStream(bOut); + out.writeObject(obj); + out.close(); + byte[] bytes = bOut.toByteArray(); + ByteArrayInputStream bIn = new ByteArrayInputStream(bytes); + ObjectInputStream in = new ObjectInputStream(bIn); + @SuppressWarnings("unchecked") + T copy = (T) in.readObject(); + in.close(); + return copy; + } +} From 4a7434884b5bff969c1c245c258bf4c6d26ee93e Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 26 Nov 2014 17:55:15 -0800 Subject: [PATCH 020/732] import change --- .../google/gcloud/datastore/KeyProperty.java | 6 ++--- .../google/gcloud/datastore/NullProperty.java | 6 ++--- .../com/google/gcloud/datastore/Property.java | 27 ++++++++++--------- .../datastore/PropertyListProperty.java | 8 +++--- .../gcloud/datastore/StringProperty.java | 6 ++--- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/KeyProperty.java b/src/main/java/com/google/gcloud/datastore/KeyProperty.java index fda5daa11dcb..3dc8debcbc79 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyProperty.java +++ b/src/main/java/com/google/gcloud/datastore/KeyProperty.java @@ -2,7 +2,7 @@ import static com.google.api.services.datastore.DatastoreV1.Value.KEY_VALUE_FIELD_NUMBER; -import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.api.services.datastore.DatastoreV1; public final class KeyProperty extends Property { @@ -22,12 +22,12 @@ public Builder newBuilder(Key key) { } @Override - protected Key getValue(Value from) { + protected Key getValue(DatastoreV1.Value from) { return Key.fromPb(from.getKeyValue()); } @Override - protected void setValue(KeyProperty from, Value.Builder to) { + protected void setValue(KeyProperty from, DatastoreV1.Value.Builder to) { to.setKeyValue(from.get().toPb()); } }; diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullProperty.java index daaaf29899cb..3ac63bd785eb 100644 --- a/src/main/java/com/google/gcloud/datastore/NullProperty.java +++ b/src/main/java/com/google/gcloud/datastore/NullProperty.java @@ -2,7 +2,7 @@ import static com.google.common.base.Preconditions.checkArgument; -import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.api.services.datastore.DatastoreV1; public final class NullProperty extends Property { @@ -22,12 +22,12 @@ public int getProtoFieldId() { } @Override - protected Void getValue(Value from) { + protected Void getValue(DatastoreV1.Value from) { return null; } @Override - protected void setValue(NullProperty from, Value.Builder to) { + protected void setValue(NullProperty from, DatastoreV1.Value.Builder to) { // nothing to set } }; diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Property.java index 50ee7ca79834..f2d3af575891 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Property.java @@ -3,7 +3,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.api.services.datastore.DatastoreV1; import com.google.protobuf.Descriptors.FieldDescriptor; import java.io.IOException; @@ -24,7 +24,7 @@ private final transient boolean indexed; private final transient Integer meaning; private final transient V value; - private transient Value tempValuePb; // only for deserialization + private transient DatastoreV1.Value tempValuePb; // only for deserialization public enum Type { @@ -54,7 +54,7 @@ public enum Type { Type(Marshaller marshaller, BuilderFactory builderFactory) { this.marshaller = marshaller; this.builderFactory = builderFactory; - field = Value.getDescriptor().findFieldByNumber(marshaller.getProtoFieldId()); + field = DatastoreV1.Value.getDescriptor().findFieldByNumber(marshaller.getProtoFieldId()); } , B extends Builder> Marshaller @@ -100,9 +100,9 @@ interface Builder, B extends Builder> { interface Marshaller, B extends Builder> { - B fromProto(Value proto); + B fromProto(DatastoreV1.Value proto); - Value toProto(P property); + DatastoreV1.Value toProto(P property); int getProtoFieldId(); } @@ -111,7 +111,7 @@ abstract static class BaseMarshaller, B extends B implements Marshaller, BuilderFactory { @Override - public final B fromProto(Value proto) { + public final B fromProto(DatastoreV1.Value proto) { B builder = newBuilder(getValue(proto)); builder.indexed(proto.getIndexed()); if (proto.hasMeaning()) { @@ -121,8 +121,8 @@ public final B fromProto(Value proto) { } @Override - public final Value toProto(P property) { - Value.Builder builder = Value.newBuilder(); + public final DatastoreV1.Value toProto(P property) { + DatastoreV1.Value.Builder builder = DatastoreV1.Value.newBuilder(); builder.setIndexed(property.getIndexed()); if (property.getMeaning() != null) { builder.setMeaning(property.getMeaning()); @@ -131,9 +131,9 @@ public final Value toProto(P property) { return builder.build(); } - protected abstract V getValue(Value from); + protected abstract V getValue(DatastoreV1.Value from); - protected abstract void setValue(P from, Value.Builder to); + protected abstract void setValue(P from, DatastoreV1.Value.Builder to); } abstract static class BaseBuilder, B extends BaseBuilder> @@ -236,6 +236,7 @@ public final V get() { return value; } + @SuppressWarnings("unchecked") public final B toBuilder() { BuilderFactory builderFactory = getType().getBuilderFactory(); B builder = builderFactory.newBuilder(get()); @@ -262,14 +263,14 @@ public boolean equals(Object other) { } @SuppressWarnings("unchecked") - Value toPb() { + DatastoreV1.Value toPb() { Marshaller marshaller = getType().getMarshaller(); return marshaller.toProto((P) this); } @SuppressWarnings({ "rawtypes", "unchecked" }) static , B extends Property.Builder> Property - fromPb(Value proto) { + fromPb(DatastoreV1.Value proto) { Marshaller marshaller = NullProperty.MARSHALLER; for (Type type : Type.values()) { FieldDescriptor descriptor = type.getDescriptor(); @@ -300,7 +301,7 @@ private void writeObject(ObjectOutputStream out) throws IOException { private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); byte[] bytes = (byte[]) in.readObject(); - tempValuePb = Value.parseFrom(bytes); + tempValuePb = DatastoreV1.Value.parseFrom(bytes); } @SuppressWarnings("unused") diff --git a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java index 9bd63168a800..5c48e50188f4 100644 --- a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java +++ b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java @@ -2,7 +2,7 @@ import static com.google.api.services.datastore.DatastoreV1.Value.LIST_VALUE_FIELD_NUMBER; -import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -28,16 +28,16 @@ public Builder newBuilder(List> properties) { } @Override - protected List> getValue(Value from) { + protected List> getValue(DatastoreV1.Value from) { List> properties = new ArrayList<>(from.getListValueCount()); - for (Value valuePb : from.getListValueList()) { + for (DatastoreV1.Value valuePb : from.getListValueList()) { properties.add(Property.fromPb(valuePb)); } return properties; } @Override - protected void setValue(PropertyListProperty from, Value.Builder to) { + protected void setValue(PropertyListProperty from, DatastoreV1.Value.Builder to) { for (Property property : from.get()) { to.addListValue(property.toPb()); } diff --git a/src/main/java/com/google/gcloud/datastore/StringProperty.java b/src/main/java/com/google/gcloud/datastore/StringProperty.java index 1b021cba0ef7..997c27cfb877 100644 --- a/src/main/java/com/google/gcloud/datastore/StringProperty.java +++ b/src/main/java/com/google/gcloud/datastore/StringProperty.java @@ -3,7 +3,7 @@ import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER; import static com.google.common.base.Preconditions.checkArgument; -import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.api.services.datastore.DatastoreV1; public final class StringProperty extends Property { @@ -23,12 +23,12 @@ public Builder newBuilder(String value) { } @Override - protected String getValue(Value from) { + protected String getValue(DatastoreV1.Value from) { return from.getStringValue(); } @Override - protected void setValue(StringProperty from, Value.Builder to) { + protected void setValue(StringProperty from, DatastoreV1.Value.Builder to) { to.setStringValue(from.get()); } }; From 221448b83dc167ce992fe7941873069ccf92e190 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 26 Nov 2014 18:15:54 -0800 Subject: [PATCH 021/732] rename property to value --- .classpath | 14 ++- .project | 4 +- .settings/org.eclipse.core.resources.prefs | 3 + .settings/org.eclipse.jdt.core.prefs | 4 + .../gcloud/datastore/EmbeddedEntity.java | 14 +-- ...Property.java => EmbeddedEntityValue.java} | 20 ++-- .../com/google/gcloud/datastore/Entity.java | 16 +-- .../{KeyProperty.java => KeyValue.java} | 18 +-- .../google/gcloud/datastore/ListValue.java | 103 +++++++++++++++++ .../{NullProperty.java => NullValue.java} | 18 +-- .../datastore/PropertyListProperty.java | 106 ------------------ .../{StringProperty.java => StringValue.java} | 18 +-- .../datastore/{Property.java => Value.java} | 58 +++++----- .../gcloud/datastore/SerializationTest.java | 52 ++++----- 14 files changed, 229 insertions(+), 219 deletions(-) create mode 100644 .settings/org.eclipse.core.resources.prefs rename src/main/java/com/google/gcloud/datastore/{EmbeddedEntityProperty.java => EmbeddedEntityValue.java} (58%) rename src/main/java/com/google/gcloud/datastore/{KeyProperty.java => KeyValue.java} (60%) create mode 100644 src/main/java/com/google/gcloud/datastore/ListValue.java rename src/main/java/com/google/gcloud/datastore/{NullProperty.java => NullValue.java} (61%) delete mode 100644 src/main/java/com/google/gcloud/datastore/PropertyListProperty.java rename src/main/java/com/google/gcloud/datastore/{StringProperty.java => StringValue.java} (63%) rename src/main/java/com/google/gcloud/datastore/{Property.java => Value.java} (78%) diff --git a/.classpath b/.classpath index 09fe957d54f4..d6cf6121af66 100644 --- a/.classpath +++ b/.classpath @@ -21,11 +21,17 @@ - - - - + + + + + + + + + + diff --git a/.project b/.project index 3b20f35b11e3..fac3e5997ca3 100644 --- a/.project +++ b/.project @@ -16,7 +16,7 @@ - org.eclipse.m2e.core.maven2Builder + edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder @@ -26,7 +26,7 @@ - edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder + org.eclipse.m2e.core.maven2Builder diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 000000000000..5b781ec6d952 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/test/java=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 0ba1de1e6e8a..fe5e054849a5 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -2,3 +2,7 @@ <<<<<<<=HEAD >>>>>>>=ef72daa81c73f99e2156f5bfe8127591fc6358e9 eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java b/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java index 853d4edf5d55..957c57ce9048 100644 --- a/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java +++ b/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java @@ -18,13 +18,13 @@ public final class EmbeddedEntity implements Serializable { private static final long serialVersionUID = 6492561268709192891L; private final transient IncompleteKey key; - private final transient ImmutableSortedMap> properties; + private final transient ImmutableSortedMap> properties; private transient DatastoreV1.Entity tempEntityPb; // only for deserialization public static final class Builder { private IncompleteKey key; - private Map> properties = new HashMap<>(); + private Map> properties = new HashMap<>(); public Builder() { } @@ -54,8 +54,8 @@ public Builder removeProperty(String name) { return this; } - public Builder setProperty(String name, Property property) { - properties.put(name, property); + public Builder setProperty(String name, Value value) { + properties.put(name, value); return this; } @@ -85,7 +85,7 @@ public boolean hasProperty(String name) { return properties.containsKey(name); } - public Property getProperty(String name) { + public Value getProperty(String name) { return properties.get(name); } @@ -119,7 +119,7 @@ static EmbeddedEntity fromPb(DatastoreV1.Entity entityPb) { builder.setKey(IncompleteKey.fromPb(entityPb.getKey())); } for (DatastoreV1.Property property : entityPb.getPropertyList()) { - builder.setProperty(property.getName(), Property.fromPb(property.getValue())); + builder.setProperty(property.getName(), Value.fromPb(property.getValue())); } return builder.build(); } @@ -129,7 +129,7 @@ DatastoreV1.Entity toPb() { if (key != null) { entityPb.setKey(key.toPb()); } - for (Map.Entry> entry : properties.entrySet()) { + for (Map.Entry> entry : properties.entrySet()) { DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); propertyPb.setName(entry.getKey()); propertyPb.setValue(entry.getValue().toPb()); diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java b/src/main/java/com/google/gcloud/datastore/EmbeddedEntityValue.java similarity index 58% rename from src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java rename to src/main/java/com/google/gcloud/datastore/EmbeddedEntityValue.java index d1d653ac9f07..d63313c6e9ff 100644 --- a/src/main/java/com/google/gcloud/datastore/EmbeddedEntityProperty.java +++ b/src/main/java/com/google/gcloud/datastore/EmbeddedEntityValue.java @@ -4,13 +4,13 @@ import com.google.api.services.datastore.DatastoreV1; -public final class EmbeddedEntityProperty extends - Property { +public final class EmbeddedEntityValue extends + Value { private static final long serialVersionUID = -5461475706792576395L; - static final BaseMarshaller MARSHALLER = - new BaseMarshaller() { + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { @Override public int getProtoFieldId() { @@ -28,13 +28,13 @@ protected EmbeddedEntity getValue(DatastoreV1.Value from) { } @Override - protected void setValue(EmbeddedEntityProperty from, DatastoreV1.Value.Builder to) { + protected void setValue(EmbeddedEntityValue from, DatastoreV1.Value.Builder to) { to.setEntityValue(from.get().toPb()); } }; public static final class Builder extends - Property.BaseBuilder { + Value.BaseBuilder { public Builder(EmbeddedEntity entity) { super(Type.EMBEDDED_ENTITY); @@ -43,16 +43,16 @@ public Builder(EmbeddedEntity entity) { } @Override - public EmbeddedEntityProperty build() { - return new EmbeddedEntityProperty(this); + public EmbeddedEntityValue build() { + return new EmbeddedEntityValue(this); } } - public EmbeddedEntityProperty(EmbeddedEntity entity) { + public EmbeddedEntityValue(EmbeddedEntity entity) { this(new Builder(entity)); } - EmbeddedEntityProperty(Builder builder) { + EmbeddedEntityValue(Builder builder) { super(builder); } } diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index 84f42b434ced..2d260b7d19b1 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -21,13 +21,13 @@ public final class Entity implements Serializable { private static final long serialVersionUID = 432961565733066915L; private final transient Key key; - private final transient ImmutableSortedMap> properties; + private final transient ImmutableSortedMap> properties; private transient DatastoreV1.Entity tempEntityPb; // only for deserialization public static final class Builder { private Key key; - private Map> properties; + private Map> properties; public Builder(Key key) { this.key = checkNotNull(key); @@ -49,8 +49,8 @@ public Builder removeProperty(String name) { return this; } - public Builder setProperty(String name, Property property) { - properties.put(name, property); + public Builder setProperty(String name, Value value) { + properties.put(name, value); return this; } @@ -72,7 +72,7 @@ public boolean hasProperty(String name) { return properties.containsKey(name); } - public Property getProperty(String name) { + public Value getProperty(String name) { return properties.get(name); } @@ -80,7 +80,7 @@ public Set getPropertyNames() { return properties.keySet(); } - ImmutableSortedMap> getProperties() { + ImmutableSortedMap> getProperties() { return properties; } @@ -108,7 +108,7 @@ static Entity fromPb(DatastoreV1.Entity entityPb) { Preconditions.checkArgument(entityPb.hasKey()); Builder builder = new Builder(Key.fromPb(entityPb.getKey())); for (DatastoreV1.Property property : entityPb.getPropertyList()) { - builder.setProperty(property.getName(), Property.fromPb(property.getValue())); + builder.setProperty(property.getName(), Value.fromPb(property.getValue())); } return builder.build(); } @@ -116,7 +116,7 @@ static Entity fromPb(DatastoreV1.Entity entityPb) { DatastoreV1.Entity toPb() { DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); entityPb.setKey(key.toPb()); - for (Map.Entry> entry : properties.entrySet()) { + for (Map.Entry> entry : properties.entrySet()) { DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); propertyPb.setName(entry.getKey()); propertyPb.setValue(entry.getValue().toPb()); diff --git a/src/main/java/com/google/gcloud/datastore/KeyProperty.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java similarity index 60% rename from src/main/java/com/google/gcloud/datastore/KeyProperty.java rename to src/main/java/com/google/gcloud/datastore/KeyValue.java index 3dc8debcbc79..637fb6931bdb 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyProperty.java +++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java @@ -4,12 +4,12 @@ import com.google.api.services.datastore.DatastoreV1; -public final class KeyProperty extends Property { +public final class KeyValue extends Value { private static final long serialVersionUID = -1318353707326704821L; - static final BaseMarshaller MARSHALLER = - new BaseMarshaller() { + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { @Override public int getProtoFieldId() { @@ -27,12 +27,12 @@ protected Key getValue(DatastoreV1.Value from) { } @Override - protected void setValue(KeyProperty from, DatastoreV1.Value.Builder to) { + protected void setValue(KeyValue from, DatastoreV1.Value.Builder to) { to.setKeyValue(from.get().toPb()); } }; - public static final class Builder extends Property.BaseBuilder { + public static final class Builder extends Value.BaseBuilder { public Builder(Key value) { super(Type.KEY); @@ -40,16 +40,16 @@ public Builder(Key value) { } @Override - public KeyProperty build() { - return new KeyProperty(this); + public KeyValue build() { + return new KeyValue(this); } } - public KeyProperty(Key key) { + public KeyValue(Key key) { this(new Builder(key)); } - KeyProperty(Builder builder) { + KeyValue(Builder builder) { super(builder); } } diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java new file mode 100644 index 000000000000..b33c096ff718 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/ListValue.java @@ -0,0 +1,103 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.LIST_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; + +public final class ListValue extends + Value>, ListValue, ListValue.Builder> { + + private static final long serialVersionUID = -5461475706792576395L; + + static final BaseMarshaller>, ListValue, Builder> MARSHALLER = + new BaseMarshaller>, ListValue, Builder>() { + + @Override + public int getProtoFieldId() { + return LIST_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(List> values) { + return new Builder().set(values); + } + + @Override + protected List> getValue(DatastoreV1.Value from) { + List> properties = new ArrayList<>(from.getListValueCount()); + for (DatastoreV1.Value valuePb : from.getListValueList()) { + properties.add(Value.fromPb(valuePb)); + } + return properties; + } + + @Override + protected void setValue(ListValue from, DatastoreV1.Value.Builder to) { + for (Value property : from.get()) { + to.addListValue(property.toPb()); + } + } + }; + + public static final class Builder extends + Value.BaseBuilder>, ListValue, Builder> { + + private ImmutableList.Builder> listBuilder = ImmutableList.builder(); + + public Builder() { + super(Type.LIST); + indexed(false); + } + + public Builder addValue(Value value) { + Preconditions.checkArgument(value.getType() != Type.LIST, "Cannot contain another list"); + listBuilder.add(value); + return this; + } + + public Builder addValue(Value first, Value... other) { + addValue(first); + for (Value value : other) { + addValue(value); + } + return this; + } + + @Override + public Builder set(List> properties) { + listBuilder = ImmutableList.>builder(); + for (Value property : properties) { + addValue(property); + } + return this; + } + + @Override + public List> get() { + return listBuilder.build(); + } + + @Override + public ListValue build() { + Preconditions.checkState(!get().isEmpty(), "value list could not be empty"); + return new ListValue(this); + } + } + + public ListValue(List> properties) { + this(new Builder().set(properties)); + } + + public ListValue(Value first, Value... other) { + this(new Builder().addValue(first, other)); + } + + ListValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/NullProperty.java b/src/main/java/com/google/gcloud/datastore/NullValue.java similarity index 61% rename from src/main/java/com/google/gcloud/datastore/NullProperty.java rename to src/main/java/com/google/gcloud/datastore/NullValue.java index 3ac63bd785eb..5d77c333a54f 100644 --- a/src/main/java/com/google/gcloud/datastore/NullProperty.java +++ b/src/main/java/com/google/gcloud/datastore/NullValue.java @@ -4,12 +4,12 @@ import com.google.api.services.datastore.DatastoreV1; -public final class NullProperty extends Property { +public final class NullValue extends Value { private static final long serialVersionUID = 8497300779013002270L; - static final BaseMarshaller MARSHALLER = - new BaseMarshaller() { + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { @Override public Builder newBuilder(Void value) { @@ -27,20 +27,20 @@ protected Void getValue(DatastoreV1.Value from) { } @Override - protected void setValue(NullProperty from, DatastoreV1.Value.Builder to) { + protected void setValue(NullValue from, DatastoreV1.Value.Builder to) { // nothing to set } }; - public static final class Builder extends Property.BaseBuilder { + public static final class Builder extends Value.BaseBuilder { public Builder() { super(Type.NULL); } @Override - public NullProperty build() { - return new NullProperty(this); + public NullValue build() { + return new NullValue(this); } @Override @@ -50,11 +50,11 @@ public Builder set(Void value) { } } - public NullProperty() { + public NullValue() { this(new Builder().indexed(true)); } - NullProperty(Builder builder) { + NullValue(Builder builder) { super(builder); } } diff --git a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java b/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java deleted file mode 100644 index 5c48e50188f4..000000000000 --- a/src/main/java/com/google/gcloud/datastore/PropertyListProperty.java +++ /dev/null @@ -1,106 +0,0 @@ -package com.google.gcloud.datastore; - -import static com.google.api.services.datastore.DatastoreV1.Value.LIST_VALUE_FIELD_NUMBER; - -import com.google.api.services.datastore.DatastoreV1; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.List; - -public final class PropertyListProperty extends - Property>, PropertyListProperty, PropertyListProperty.Builder> { - - private static final long serialVersionUID = -5461475706792576395L; - - static final BaseMarshaller>, PropertyListProperty, Builder> MARSHALLER = - new BaseMarshaller>, PropertyListProperty, Builder>() { - - @Override - public int getProtoFieldId() { - return LIST_VALUE_FIELD_NUMBER; - } - - @Override - public Builder newBuilder(List> properties) { - return new Builder().set(properties); - } - - @Override - protected List> getValue(DatastoreV1.Value from) { - List> properties = new ArrayList<>(from.getListValueCount()); - for (DatastoreV1.Value valuePb : from.getListValueList()) { - properties.add(Property.fromPb(valuePb)); - } - return properties; - } - - @Override - protected void setValue(PropertyListProperty from, DatastoreV1.Value.Builder to) { - for (Property property : from.get()) { - to.addListValue(property.toPb()); - } - } - }; - - public static final class Builder extends - Property.BaseBuilder>, PropertyListProperty, Builder> { - - private ImmutableList.Builder> listBuilder = ImmutableList.builder(); - - public Builder() { - super(Type.PROPERTY_LIST); - indexed(false); - } - - public Builder addProperty(Property property) { - Preconditions.checkArgument( - property.getType() != Type.PROPERTY_LIST, "Cannot contain another list"); - listBuilder.add(property); - return this; - } - - public Builder addProperty(Property first, Property... other) { - addProperty(first); - for (Property property : other) { - Preconditions.checkArgument( - property.getType() != Type.PROPERTY_LIST, "Cannot contain another list"); - listBuilder.add(property); - } - return this; - } - - @Override - public Builder set(List> properties) { - listBuilder = ImmutableList.>builder(); - for (Property property : properties) { - addProperty(property); - } - return this; - } - - @Override - public List> get() { - return listBuilder.build(); - } - - @Override - public PropertyListProperty build() { - Preconditions.checkState(!get().isEmpty(), "Property list could not be empty"); - return new PropertyListProperty(this); - } - } - - public PropertyListProperty(List> properties) { - this(new Builder().set(properties)); - } - - public PropertyListProperty(Property first, Property... other) { - this(new Builder().addProperty(first, other)); - } - - PropertyListProperty(Builder builder) { - super(builder); - } -} diff --git a/src/main/java/com/google/gcloud/datastore/StringProperty.java b/src/main/java/com/google/gcloud/datastore/StringValue.java similarity index 63% rename from src/main/java/com/google/gcloud/datastore/StringProperty.java rename to src/main/java/com/google/gcloud/datastore/StringValue.java index 997c27cfb877..70d77e250782 100644 --- a/src/main/java/com/google/gcloud/datastore/StringProperty.java +++ b/src/main/java/com/google/gcloud/datastore/StringValue.java @@ -5,12 +5,12 @@ import com.google.api.services.datastore.DatastoreV1; -public final class StringProperty extends Property { +public final class StringValue extends Value { private static final long serialVersionUID = -3105699707394545523L; - static final BaseMarshaller MARSHALLER = - new BaseMarshaller() { + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { @Override public int getProtoFieldId() { @@ -28,12 +28,12 @@ protected String getValue(DatastoreV1.Value from) { } @Override - protected void setValue(StringProperty from, DatastoreV1.Value.Builder to) { + protected void setValue(StringValue from, DatastoreV1.Value.Builder to) { to.setStringValue(from.get()); } }; - public static final class Builder extends Property.BaseBuilder { + public static final class Builder extends Value.BaseBuilder { public Builder(String value) { super(Type.STRING); @@ -41,19 +41,19 @@ public Builder(String value) { } @Override - public StringProperty build() { + public StringValue build() { if (getIndexed()) { checkArgument(get().length() <= 500, "Indexed string is limited to 500 characters"); } - return new StringProperty(this); + return new StringValue(this); } } - public StringProperty(String content) { + public StringValue(String content) { this(new Builder(content)); } - StringProperty(Builder builder) { + StringValue(Builder builder) { super(builder); } } diff --git a/src/main/java/com/google/gcloud/datastore/Property.java b/src/main/java/com/google/gcloud/datastore/Value.java similarity index 78% rename from src/main/java/com/google/gcloud/datastore/Property.java rename to src/main/java/com/google/gcloud/datastore/Value.java index f2d3af575891..2a685eaf5acd 100644 --- a/src/main/java/com/google/gcloud/datastore/Property.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -15,7 +15,7 @@ // TODO: add javadoc, and mention that null should only be represented by NullValue. public abstract class - Property, B extends Property.Builder> + Value, B extends Value.Builder> implements Serializable { private static final long serialVersionUID = -1899638277588872742L; @@ -28,11 +28,11 @@ public enum Type { - NULL(NullProperty.MARSHALLER, NullProperty.MARSHALLER), - STRING(StringProperty.MARSHALLER, StringProperty.MARSHALLER), - EMBEDDED_ENTITY(EmbeddedEntityProperty.MARSHALLER, EmbeddedEntityProperty.MARSHALLER), - PROPERTY_LIST(PropertyListProperty.MARSHALLER, PropertyListProperty.MARSHALLER), - KEY(KeyProperty.MARSHALLER, KeyProperty.MARSHALLER); + NULL(NullValue.MARSHALLER, NullValue.MARSHALLER), + STRING(StringValue.MARSHALLER, StringValue.MARSHALLER), + EMBEDDED_ENTITY(EmbeddedEntityValue.MARSHALLER, EmbeddedEntityValue.MARSHALLER), + LIST(ListValue.MARSHALLER, ListValue.MARSHALLER), + KEY(KeyValue.MARSHALLER, KeyValue.MARSHALLER); /* TODO: Also implement @@ -50,19 +50,19 @@ public enum Type { @SuppressWarnings("rawtypes") private final Marshaller marshaller; private FieldDescriptor field; - , B extends Builder> + , B extends Builder> Type(Marshaller marshaller, BuilderFactory builderFactory) { this.marshaller = marshaller; this.builderFactory = builderFactory; field = DatastoreV1.Value.getDescriptor().findFieldByNumber(marshaller.getProtoFieldId()); } - , B extends Builder> Marshaller + , B extends Builder> Marshaller getMarshaller() { return marshaller; } - , B extends Builder> BuilderFactory + , B extends Builder> BuilderFactory getBuilderFactory() { return builderFactory; } @@ -72,12 +72,12 @@ FieldDescriptor getDescriptor() { } } - interface BuilderFactory, B extends Builder> { + interface BuilderFactory, B extends Builder> { B newBuilder(V value); } - interface Builder, B extends Builder> { + interface Builder, B extends Builder> { Type getType(); @@ -98,16 +98,16 @@ interface Builder, B extends Builder> { P build(); } - interface Marshaller, B extends Builder> { + interface Marshaller, B extends Builder> { B fromProto(DatastoreV1.Value proto); - DatastoreV1.Value toProto(P property); + DatastoreV1.Value toProto(P value); int getProtoFieldId(); } - abstract static class BaseMarshaller, B extends Builder> + abstract static class BaseMarshaller, B extends Builder> implements Marshaller, BuilderFactory { @Override @@ -121,13 +121,13 @@ public final B fromProto(DatastoreV1.Value proto) { } @Override - public final DatastoreV1.Value toProto(P property) { + public final DatastoreV1.Value toProto(P value) { DatastoreV1.Value.Builder builder = DatastoreV1.Value.newBuilder(); - builder.setIndexed(property.getIndexed()); - if (property.getMeaning() != null) { - builder.setMeaning(property.getMeaning()); + builder.setIndexed(value.getIndexed()); + if (value.getMeaning() != null) { + builder.setMeaning(value.getMeaning()); } - setValue(property, builder); + setValue(value, builder); return builder.build(); } @@ -136,7 +136,7 @@ public final DatastoreV1.Value toProto(P property) { protected abstract void setValue(P from, DatastoreV1.Value.Builder to); } - abstract static class BaseBuilder, B extends BaseBuilder> + abstract static class BaseBuilder, B extends BaseBuilder> implements Builder { private final Type type; @@ -203,7 +203,7 @@ private B self() { public abstract P build(); } - Property(Builder builder) { + Value(Builder builder) { type = builder.getType(); indexed = builder.getIndexed(); meaning = builder.getMeaning(); @@ -255,11 +255,11 @@ public boolean equals(Object other) { return false; } - Property otherProperty = (Property) other; - return Objects.equals(type, otherProperty.getType()) - && Objects.equals(indexed, otherProperty.getIndexed()) - && Objects.equals(meaning, otherProperty.getMeaning()) - && Objects.equals(value, otherProperty.get()); + Value otherValue = (Value) other; + return Objects.equals(type, otherValue.getType()) + && Objects.equals(indexed, otherValue.getIndexed()) + && Objects.equals(meaning, otherValue.getMeaning()) + && Objects.equals(value, otherValue.get()); } @SuppressWarnings("unchecked") @@ -269,9 +269,9 @@ DatastoreV1.Value toPb() { } @SuppressWarnings({ "rawtypes", "unchecked" }) - static , B extends Property.Builder> Property + static , B extends Value.Builder> Value fromPb(DatastoreV1.Value proto) { - Marshaller marshaller = NullProperty.MARSHALLER; + Marshaller marshaller = NullValue.MARSHALLER; for (Type type : Type.values()) { FieldDescriptor descriptor = type.getDescriptor(); if (descriptor != null) { @@ -286,7 +286,7 @@ DatastoreV1.Value toPb() { } } } - // change strategy to return RawProperty (package scope constructor) + // change strategy to return RawValue (package scope constructor) // when no match instead of null. This could only be done // when using the V3 API which added a NullValue type to distinct the cases // and the use of oneof which generates an enum of all possible values. diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 19971da7af6d..48a91eceb8e4 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -3,7 +3,7 @@ import static org.junit.Assert.assertEquals; import com.google.common.collect.ImmutableMap; -import com.google.gcloud.datastore.Property.Type; +import com.google.gcloud.datastore.Value.Type; import org.junit.Test; @@ -19,13 +19,13 @@ public class SerializationTest { new IncompleteKey.Builder("ds", "k").addToPath("p", 1).build(); private static final Key KEY1 = new Key.Builder("ds", "k", "n").build(); private static final Key KEY2 = new Key.Builder(INCOMPLETE_KEY, 2).build(); - private static final KeyProperty KEY_PROPERTY = new KeyProperty(KEY1); - private static final NullProperty NULL_PROPERTY = - new NullProperty.Builder().indexed(true).build(); - private static final StringProperty STRING_PROPERTY = new StringProperty("hello"); + private static final KeyValue KEY_PROPERTY = new KeyValue(KEY1); + private static final NullValue NULL_PROPERTY = + new NullValue.Builder().indexed(true).build(); + private static final StringValue STRING_PROPERTY = new StringValue("hello"); private static final Entity ENTITY1 = new Entity.Builder(KEY1).build(); private static final Entity ENTITY2 = - new Entity.Builder(KEY2).setProperty("null", new NullProperty()).build(); + new Entity.Builder(KEY2).setProperty("null", new NullValue()).build(); private static final EmbeddedEntity EMBEDDED_ENTITY1 = new EmbeddedEntity(ENTITY1); private static final EmbeddedEntity EMBEDDED_ENTITY2 = new EmbeddedEntity(ENTITY2); private static final EmbeddedEntity EMBEDDED_ENTITY3 = @@ -34,46 +34,46 @@ public class SerializationTest { .setProperty("p1", STRING_PROPERTY) .setProperty("p2", STRING_PROPERTY) .build(); - private static final EmbeddedEntityProperty EMBEDDED_ENTITY_PROPERTY1 = - new EmbeddedEntityProperty(EMBEDDED_ENTITY1); - private static final EmbeddedEntityProperty EMBEDDED_ENTITY_PROPERTY2 = - new EmbeddedEntityProperty(EMBEDDED_ENTITY2); - private static final EmbeddedEntityProperty EMBEDDED_ENTITY_PROPERTY3 = - new EmbeddedEntityProperty(EMBEDDED_ENTITY3); - private static final PropertyListProperty PROPERTY_LIST_PROPERTY = - new PropertyListProperty.Builder() - .addProperty(NULL_PROPERTY) - .addProperty(STRING_PROPERTY) - .addProperty(new NullProperty()) + private static final EmbeddedEntityValue EMBEDDED_ENTITY_PROPERTY1 = + new EmbeddedEntityValue(EMBEDDED_ENTITY1); + private static final EmbeddedEntityValue EMBEDDED_ENTITY_PROPERTY2 = + new EmbeddedEntityValue(EMBEDDED_ENTITY2); + private static final EmbeddedEntityValue EMBEDDED_ENTITY_PROPERTY3 = + new EmbeddedEntityValue(EMBEDDED_ENTITY3); + private static final ListValue PROPERTY_LIST_PROPERTY = + new ListValue.Builder() + .addValue(NULL_PROPERTY) + .addValue(STRING_PROPERTY) + .addValue(new NullValue()) .build(); - private Map typeToProperties = ImmutableMap.of( + private Map typeToValues = ImmutableMap.of( Type.NULL, array(NULL_PROPERTY), Type.KEY, array(KEY_PROPERTY), Type.STRING, array(STRING_PROPERTY), Type.EMBEDDED_ENTITY, array(EMBEDDED_ENTITY_PROPERTY1, EMBEDDED_ENTITY_PROPERTY2, EMBEDDED_ENTITY_PROPERTY3), - Type.PROPERTY_LIST, array(PROPERTY_LIST_PROPERTY)); + Type.LIST, array(PROPERTY_LIST_PROPERTY)); private static T[] array(T... values) { return values; } @Test - public void testProperties() throws Exception { + public void testValues() throws Exception { for (Type type : Type.values()) { - Property[] properties = typeToProperties.get(type); - for (Property property : properties) { - Property copy = serialiazeAndDeserialize(property); - assertEquals(property, copy); - assertEquals(property.get(), copy.get()); + Value[] values = typeToValues.get(type); + for (Value value : values) { + Value copy = serialiazeAndDeserialize(value); + assertEquals(value, copy); + assertEquals(value.get(), copy.get()); } } } @Test public void testTypes() throws Exception { - Object[] types = new Object[] { KEY1, KEY2, INCOMPLETE_KEY, ENTITY1, ENTITY2, EMBEDDED_ENTITY1, + Object[] types = { KEY1, KEY2, INCOMPLETE_KEY, ENTITY1, ENTITY2, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; for (Object obj : types) { Object copy = serialiazeAndDeserialize(obj); From 65a9f121fbe37a34b1f855e4505853dba26d7a1d Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 26 Nov 2014 18:20:17 -0800 Subject: [PATCH 022/732] bla --- .../java/com/google/gcloud/datastore/DatastoreService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index ca4c55033322..604f69b0620f 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -5,7 +5,8 @@ public interface DatastoreService extends DatastoreReader, DatastoreWriter { interface Query { - + // TODO + // consider 2 types of queries regualr and Gql } From aa237dcf201e5865535d68689092cb6e6b220326 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 1 Dec 2014 17:37:14 -0800 Subject: [PATCH 023/732] work in progress --- .../gcloud/datastore/BatchWriteOption.java | 31 +++++ .../google/gcloud/datastore/BatchWriter.java | 6 + .../gcloud/datastore/DatastoreService.java | 65 +++++------ .../datastore/DatastoreServiceException.java | 20 ++++ .../datastore/DatastoreServiceImpl.java | 13 ++- .../datastore/DatastoreServiceOptions.java | 13 +++ .../gcloud/datastore/EmbeddedEntityValue.java | 58 ---------- .../com/google/gcloud/datastore/Entity.java | 107 ++++-------------- .../java/com/google/gcloud/datastore/Key.java | 43 ++++--- .../google/gcloud/datastore/KeyBuilder.java | 40 +++++++ ...EmbeddedEntity.java => PartialEntity.java} | 53 ++++----- .../gcloud/datastore/PartialEntityValue.java | 58 ++++++++++ .../{IncompleteKey.java => PartialKey.java} | 18 +-- .../google/gcloud/datastore/Transaction.java | 8 ++ .../gcloud/datastore/TransactionOption.java | 59 ++++++++++ .../com/google/gcloud/datastore/Value.java | 2 +- .../datastore/DatastoreServiceTest.java | 69 +++++++++++ .../gcloud/datastore/SerializationTest.java | 26 ++--- 18 files changed, 433 insertions(+), 256 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/BatchWriteOption.java create mode 100644 src/main/java/com/google/gcloud/datastore/BatchWriter.java create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java delete mode 100644 src/main/java/com/google/gcloud/datastore/EmbeddedEntityValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/KeyBuilder.java rename src/main/java/com/google/gcloud/datastore/{EmbeddedEntity.java => PartialEntity.java} (71%) create mode 100644 src/main/java/com/google/gcloud/datastore/PartialEntityValue.java rename src/main/java/com/google/gcloud/datastore/{IncompleteKey.java => PartialKey.java} (94%) create mode 100644 src/main/java/com/google/gcloud/datastore/Transaction.java create mode 100644 src/main/java/com/google/gcloud/datastore/TransactionOption.java create mode 100644 src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java new file mode 100644 index 000000000000..cda7bf7be16f --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java @@ -0,0 +1,31 @@ +package com.google.gcloud.datastore; + +import java.io.Serializable; + +public abstract class BatchWriteOption implements Serializable { + + private static final long serialVersionUID = -3932758377282659839L; + + BatchWriteOption() { + // package protected + } + + public static final class ForceWrites extends BatchWriteOption { + + private static final long serialVersionUID = 2555054296046232799L; + + private final boolean force; + + public ForceWrites(boolean force) { + this.force = force; + } + + public boolean isForce() { + return force; + } + } + + public static ForceWrites forceWrites() { + return new ForceWrites(true); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java new file mode 100644 index 000000000000..9d2e74f01cb1 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java @@ -0,0 +1,6 @@ +package com.google.gcloud.datastore; + +public interface BatchWriter extends DatastoreWriter { + + void submit(); +} \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 604f69b0620f..88993173bbb8 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -4,47 +4,48 @@ public interface DatastoreService extends DatastoreReader, DatastoreWriter { - interface Query { - // TODO - // consider 2 types of queries regualr and Gql + enum QueryType { + PROJECTION, + FULL; } + /* + interface QueryResult { - public interface Transaction extends DatastoreReader, DatastoreWriter { - - void commit(); - - void rollback(); } - public interface TransactionOptions { - - enum IsolationLevel { - SERIALIZABLE, SNAPSHOT; - } - - - IsolationLevel getIsolationLevel(); - - boolean force(); - } - - public interface BatchWriter extends DatastoreWriter { - - void submit(); + // consider 2 types of queries regualr and Gql + interface Query { + QueryResult keysOnly(QueryType query); } +*/ - public interface BatchOptions { - } DatastoreServiceOptions getOptions(); - Transaction newTransaction(TransactionOptions transactionOptions); - - BatchWriter newBatchWriter(BatchOptions batchOptions); - - Key allocateId(IncompleteKey key); - + KeyBuilder newKeyBuilder(String kind); + + /** + * Returns a new Datastore transaction. + * + * @throws DatastoreServiceExcepiton upon failure + */ + Transaction newTransaction(TransactionOption... transactionOption); + + BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption); + + /** + * Returns a key with a newly allocated id. + * + * @throws DatastoreServiceExcepiton upon failure + */ + Key allocateId(PartialKey key); + + /** + * Returns a list of keys using the allocated ids ordered by the input. + * + * @throws DatastoreServiceExcepiton upon failure + */ // results are returned using request order - Iterator allocateIds(IncompleteKey... key); + Iterator allocateIds(PartialKey... key); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java new file mode 100644 index 000000000000..5b27cd519654 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java @@ -0,0 +1,20 @@ +package com.google.gcloud.datastore; + +public class DatastoreServiceException extends RuntimeException { + + private static final long serialVersionUID = 8170357898917041899L; + + private final boolean isTransient; + + public DatastoreServiceException(boolean isTransient, String msg, Exception cause) { + super(msg, cause); + this.isTransient = isTransient; + } + + /** + * @return {@code true} if this exception is transient and could be safely retried. + */ + public boolean isTransient() { + return isTransient; + } +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index cc5ed9fa3434..cd5475c12bab 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -17,25 +17,25 @@ public DatastoreServiceOptions getOptions() { } @Override - public Transaction newTransaction(TransactionOptions transactionOptions) { + public Transaction newTransaction(TransactionOption... transactionOption) { // TODO Auto-generated method stub return null; } @Override - public BatchWriter newBatchWriter(BatchOptions batchOptions) { + public BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption) { // TODO Auto-generated method stub return null; } @Override - public Key allocateId(IncompleteKey key) { + public Key allocateId(PartialKey key) { // TODO Auto-generated method stub return null; } @Override - public Iterator allocateIds(IncompleteKey... key) { + public Iterator allocateIds(PartialKey... key) { // TODO Auto-generated method stub return null; } @@ -75,4 +75,9 @@ public void delete(Key... key) { // TODO Auto-generated method stub } + + @Override + public KeyBuilder newKeyBuilder(String kind) { + return new KeyBuilder(kind, options.getDefaultNamespace(), kind); + } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index bf2a58978fc1..06377531b25a 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableSet; import com.google.gcloud.ServiceOptions; +import java.lang.reflect.Method; import java.util.Set; import java.util.regex.Pattern; @@ -72,6 +73,18 @@ public String getDataset() { return dataset; } + public String getDefaultNamespace() { + // TODO(ozarov): An alternative to reflection would be to depend on AE api jar: + // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0 + try { + Class clazz = Class.forName("com.google.appengine.api.NamespaceManager"); + Method method = clazz.getMethod("get"); + return (String) method.invoke(null); + } catch (Exception ex) { + return ""; + } + } + public boolean getForce() { return force; } diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntityValue.java b/src/main/java/com/google/gcloud/datastore/EmbeddedEntityValue.java deleted file mode 100644 index d63313c6e9ff..000000000000 --- a/src/main/java/com/google/gcloud/datastore/EmbeddedEntityValue.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.google.gcloud.datastore; - -import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; - -import com.google.api.services.datastore.DatastoreV1; - -public final class EmbeddedEntityValue extends - Value { - - private static final long serialVersionUID = -5461475706792576395L; - - static final BaseMarshaller MARSHALLER = - new BaseMarshaller() { - - @Override - public int getProtoFieldId() { - return ENTITY_VALUE_FIELD_NUMBER; - } - - @Override - public Builder newBuilder(EmbeddedEntity value) { - return new Builder(value); - } - - @Override - protected EmbeddedEntity getValue(DatastoreV1.Value from) { - return EmbeddedEntity.fromPb(from.getEntityValue()); - } - - @Override - protected void setValue(EmbeddedEntityValue from, DatastoreV1.Value.Builder to) { - to.setEntityValue(from.get().toPb()); - } - }; - - public static final class Builder extends - Value.BaseBuilder { - - public Builder(EmbeddedEntity entity) { - super(Type.EMBEDDED_ENTITY); - indexed(false); - set(entity); - } - - @Override - public EmbeddedEntityValue build() { - return new EmbeddedEntityValue(this); - } - } - - public EmbeddedEntityValue(EmbeddedEntity entity) { - this(new Builder(entity)); - } - - EmbeddedEntityValue(Builder builder) { - super(builder); - } -} diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index 2d260b7d19b1..7673015bb227 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -6,102 +6,56 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSortedMap; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.ObjectStreamException; -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -public final class Entity implements Serializable { +public final class Entity extends PartialEntity { private static final long serialVersionUID = 432961565733066915L; - private final transient Key key; - private final transient ImmutableSortedMap> properties; - private transient DatastoreV1.Entity tempEntityPb; // only for deserialization - public static final class Builder { - private Key key; - private Map> properties; + private PartialEntity.Builder delegate; public Builder(Key key) { - this.key = checkNotNull(key); - this.properties = new HashMap<>(); + delegate = new PartialEntity.Builder(); + delegate.setKey(checkNotNull(key)); } public Builder(Entity entity) { - this.key = entity.key; - this.properties = new HashMap<>(entity.properties); + delegate = new PartialEntity.Builder(entity); } public Builder clearProperties() { - properties.clear(); + delegate.clearProperties(); return this; } public Builder removeProperty(String name) { - properties.remove(name); + delegate.removeProperty(name); return this; } public Builder setProperty(String name, Value value) { - properties.put(name, value); + delegate.setProperty(name, value); return this; } public Entity build() { - return new Entity(this); + PartialEntity entity = delegate.build(); + return new Entity((Key) entity.getKey(), entity.getProperties()); } } - private Entity(Builder builder) { - key = builder.key; - properties = ImmutableSortedMap.copyOf(builder.properties); - } - - public Key getKey() { - return key; - } - - public boolean hasProperty(String name) { - return properties.containsKey(name); - } - - public Value getProperty(String name) { - return properties.get(name); - } - - public Set getPropertyNames() { - return properties.keySet(); - } - - ImmutableSortedMap> getProperties() { - return properties; - } - - @Override - public String toString() { - return toPb().toString(); - } - - @Override - public int hashCode() { - return Objects.hash(key, properties); + private Entity(Key key, ImmutableSortedMap> properties) { + super(key, properties); } + /** + * Returns the entity's key (never null). + */ @Override - public boolean equals(Object obj) { - if (!(obj instanceof Entity)) { - return false; - } - Entity other = (Entity) obj; - return Objects.equals(key, other.key) - && Objects.equals(properties, other.properties); + public Key getKey() { + return (Key) super.getKey(); } static Entity fromPb(DatastoreV1.Entity entityPb) { @@ -113,31 +67,8 @@ static Entity fromPb(DatastoreV1.Entity entityPb) { return builder.build(); } - DatastoreV1.Entity toPb() { - DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); - entityPb.setKey(key.toPb()); - for (Map.Entry> entry : properties.entrySet()) { - DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); - propertyPb.setName(entry.getKey()); - propertyPb.setValue(entry.getValue().toPb()); - entityPb.addProperty(propertyPb.build()); - } - return entityPb.build(); - } - - private void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - out.writeObject(toPb().toByteArray()); - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - in.defaultReadObject(); - byte[] bytes = (byte[]) in.readObject(); - tempEntityPb = DatastoreV1.Entity.parseFrom(bytes); - } - - @SuppressWarnings("unused") - private Object readResolve() throws ObjectStreamException { + @Override + protected Object readResolve() throws ObjectStreamException { return fromPb(tempEntityPb); } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index adad267b27b2..bc145a157b7b 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -16,43 +16,43 @@ /** * A key that is guaranteed to be complete. */ -public final class Key extends IncompleteKey { +public final class Key extends PartialKey { private static final long serialVersionUID = 3160994559785491356L; public static class Builder { - private final IncompleteKey.Builder keyBuilder; + private final PartialKey.Builder delegate; private String name; private Long id; public Builder(String dataset, String kind, String name) { - keyBuilder = new IncompleteKey.Builder(dataset, kind); + delegate = new PartialKey.Builder(dataset, kind); this.name = name; } public Builder(String dataset, String kind, long id) { - keyBuilder = new IncompleteKey.Builder(dataset, kind); + delegate = new PartialKey.Builder(dataset, kind); this.id = id; } - public Builder(IncompleteKey key, String name) { - keyBuilder = new IncompleteKey.Builder(key); + public Builder(PartialKey key, String name) { + delegate = new PartialKey.Builder(key); this.name = name; } - public Builder(IncompleteKey key, long id) { - keyBuilder = new IncompleteKey.Builder(key); + public Builder(PartialKey key, long id) { + delegate = new PartialKey.Builder(key); this.id = id; } public Builder setDataset(String dataset) { - keyBuilder.setDataset(checkNotNull(dataset)); + delegate.setDataset(checkNotNull(dataset)); return this; } public Builder setNamespace(String namespace) { - keyBuilder.setNamespace(checkNotNull(namespace)); + delegate.setNamespace(checkNotNull(namespace)); return this; } @@ -69,31 +69,31 @@ public Builder setId(long id) { } public Builder addToPath(String kind, long id) { - keyBuilder.addToPath(kind, id); + delegate.addToPath(kind, id); return this; } public Builder addToPath(String kind, String name) { - keyBuilder.addToPath(kind, name); + delegate.addToPath(kind, name); return this; } public Builder clearPath() { - keyBuilder.clearPath(); + delegate.clearPath(); return this; } public Key build() { - IncompleteKey key = keyBuilder.build(); + PartialKey key = delegate.build(); return id == null ? new Key(key, name) : new Key(key, id); } } - private Key(IncompleteKey key, String name) { + private Key(PartialKey key, String name) { super(key.getDataset(), key.getNamespace(), newPath(key, name)); } - private Key(IncompleteKey key, long id) { + private Key(PartialKey key, long id) { super(key.getDataset(), key.getNamespace(), newPath(key, id)); } @@ -136,7 +136,7 @@ public static Key fromUrlSafe(String urlSafe) { * @throws IllegalArgumentException if provided key is not complete. */ - public static Key fromIncompleteKey(IncompleteKey key) { + public static Key fromIncompleteKey(PartialKey key) { if (key instanceof Key) { return (Key) key; } @@ -150,23 +150,22 @@ public static Key fromIncompleteKey(IncompleteKey key) { } static Key fromPb(DatastoreV1.Key keyPb) { - return fromIncompleteKey(IncompleteKey.fromPb(keyPb)); + return fromIncompleteKey(PartialKey.fromPb(keyPb)); } @Override - @SuppressWarnings("unused") protected Object readResolve() throws ObjectStreamException { - return fromIncompleteKey((IncompleteKey) super.readResolve()); + return fromIncompleteKey((PartialKey) super.readResolve()); } - private static ImmutableList newPath(IncompleteKey key, String name) { + private static ImmutableList newPath(PartialKey key, String name) { ImmutableList.Builder path = ImmutableList.builder(); path.addAll(key.getAncestorPath()); path.add(new PathElement(key.getKind(), name)); return path.build(); } - private static ImmutableList newPath(IncompleteKey key, long id) { + private static ImmutableList newPath(PartialKey key, long id) { ImmutableList.Builder path = ImmutableList.builder(); path.addAll(key.getAncestorPath()); path.add(new PathElement(key.getKind(), id)); diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java new file mode 100644 index 000000000000..dcc5fecad742 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -0,0 +1,40 @@ +package com.google.gcloud.datastore; + +public final class KeyBuilder { + + private final PartialKey.Builder delegate; + + public KeyBuilder(String dataset, String namespace, String kind) { + delegate = new PartialKey.Builder(dataset, kind); + if (namespace != null) { + delegate.setNamespace(namespace); + } + } + + public KeyBuilder addToPath(String kind, String name) { + delegate.addToPath(kind, name); + return this; + } + + public KeyBuilder addToPath(String kind, long id) { + delegate.addToPath(kind, id); + return this; + } + + public KeyBuilder setNamespace(String namespace) { + delegate.setNamespace(namespace); + return this; + } + + public PartialKey build() { + return delegate.build(); + } + + public Key build(String name) { + return new Key.Builder(build(), name).build(); + } + + public Key build(long id) { + return new Key.Builder(build(), id).build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java similarity index 71% rename from src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java rename to src/main/java/com/google/gcloud/datastore/PartialEntity.java index 957c57ce9048..b96e4ad7b6e9 100644 --- a/src/main/java/com/google/gcloud/datastore/EmbeddedEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -13,33 +13,29 @@ import java.util.Objects; import java.util.Set; -public final class EmbeddedEntity implements Serializable { +public class PartialEntity implements Serializable { private static final long serialVersionUID = 6492561268709192891L; - private final transient IncompleteKey key; + private final transient PartialKey key; private final transient ImmutableSortedMap> properties; - private transient DatastoreV1.Entity tempEntityPb; // only for deserialization + protected transient DatastoreV1.Entity tempEntityPb; // only for deserialization public static final class Builder { - private IncompleteKey key; - private Map> properties = new HashMap<>(); + private PartialKey key; + private Map> properties; public Builder() { + properties = new HashMap<>(); } - public Builder(EmbeddedEntity entity) { - this.key = entity.key; - this.properties = new HashMap<>(entity.properties); - } - - public Builder(Entity entity) { + public Builder(PartialEntity entity) { this.key = entity.getKey(); this.properties = new HashMap<>(entity.getProperties()); } - public Builder setKey(IncompleteKey key) { + public Builder setKey(PartialKey key) { this.key = key; return this; } @@ -59,25 +55,20 @@ public Builder setProperty(String name, Value value) { return this; } - public EmbeddedEntity build() { - return new EmbeddedEntity(this); + public PartialEntity build() { + return new PartialEntity(key, ImmutableSortedMap.copyOf(properties)); } } - private EmbeddedEntity(Builder builder) { - key = builder.key; - properties = ImmutableSortedMap.copyOf(builder.properties); - } - - public EmbeddedEntity(Entity entity) { - key = entity.getKey(); - properties = entity.getProperties(); + protected PartialEntity(PartialKey key, ImmutableSortedMap> properties) { + this.key = key; + this.properties = properties; } /** * Returns the key or null if not provided. */ - public IncompleteKey getKey() { + public PartialKey getKey() { return key; } @@ -105,18 +96,22 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof EmbeddedEntity)) { + if (!(obj instanceof PartialEntity)) { return false; } - EmbeddedEntity other = (EmbeddedEntity) obj; + PartialEntity other = (PartialEntity) obj; return Objects.equals(key, other.key) && Objects.equals(properties, other.properties); } - static EmbeddedEntity fromPb(DatastoreV1.Entity entityPb) { + ImmutableSortedMap> getProperties() { + return properties; + } + + static PartialEntity fromPb(DatastoreV1.Entity entityPb) { Builder builder = new Builder(); if (entityPb.hasKey()) { - builder.setKey(IncompleteKey.fromPb(entityPb.getKey())); + builder.setKey(PartialKey.fromPb(entityPb.getKey())); } for (DatastoreV1.Property property : entityPb.getPropertyList()) { builder.setProperty(property.getName(), Value.fromPb(property.getValue())); @@ -124,7 +119,7 @@ static EmbeddedEntity fromPb(DatastoreV1.Entity entityPb) { return builder.build(); } - DatastoreV1.Entity toPb() { + protected DatastoreV1.Entity toPb() { DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); if (key != null) { entityPb.setKey(key.toPb()); @@ -150,7 +145,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE } @SuppressWarnings("unused") - private Object readResolve() throws ObjectStreamException { + protected Object readResolve() throws ObjectStreamException { return fromPb(tempEntityPb); } } diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java b/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java new file mode 100644 index 000000000000..db076a3305a0 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java @@ -0,0 +1,58 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public class PartialEntityValue extends + Value { + + private static final long serialVersionUID = -5461475706792576395L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return ENTITY_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(PartialEntity value) { + return new Builder(value); + } + + @Override + protected PartialEntity getValue(DatastoreV1.Value from) { + return PartialEntity.fromPb(from.getEntityValue()); + } + + @Override + protected void setValue(PartialEntityValue from, DatastoreV1.Value.Builder to) { + to.setEntityValue(from.get().toPb()); + } + }; + + public static final class Builder extends + Value.BaseBuilder { + + public Builder(PartialEntity entity) { + super(Type.PARTIAL_ENTITY); + indexed(false); + set(entity); + } + + @Override + public PartialEntityValue build() { + return new PartialEntityValue(this); + } + } + + public PartialEntityValue(PartialEntity entity) { + this(new Builder(entity)); + } + + PartialEntityValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java similarity index 94% rename from src/main/java/com/google/gcloud/datastore/IncompleteKey.java rename to src/main/java/com/google/gcloud/datastore/PartialKey.java index 8e67a39b5f0d..29856f7e11c0 100644 --- a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -19,7 +19,7 @@ import java.util.List; import java.util.Objects; -public class IncompleteKey implements Serializable { +public class PartialKey implements Serializable { private static final long serialVersionUID = -75301206578793347L; @@ -151,7 +151,7 @@ public Builder(String dataset, String kind) { this.kind = validateKind(kind); } - public Builder(IncompleteKey key) { + public Builder(PartialKey key) { dataset = key.dataset; namespace = key.namespace; kind = key.getKind(); @@ -201,15 +201,15 @@ public Builder setNamespace(String namespace) { return this; } - public IncompleteKey build() { + public PartialKey build() { PathElement leaf = new PathElement(kind); ImmutableList pathList = ImmutableList.builder().addAll(path).add(leaf).build(); - return new IncompleteKey(dataset, namespace, pathList); + return new PartialKey(dataset, namespace, pathList); } } - IncompleteKey(String dataset, String namespace, ImmutableList path) { + PartialKey(String dataset, String namespace, ImmutableList path) { checkState(!path.isEmpty(), "path must not be empty"); this.dataset = dataset; this.namespace = namespace; @@ -244,10 +244,10 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (!(other instanceof IncompleteKey)) { + if (!(other instanceof PartialKey)) { return false; } - IncompleteKey otherKey = (IncompleteKey) other; + PartialKey otherKey = (PartialKey) other; return Objects.equals(dataset, otherKey.dataset) && Objects.equals(namespace, otherKey.namespace) && Objects.equals(path, otherKey.path); @@ -257,7 +257,7 @@ PathElement getLeaf() { return path.get(path.size() - 1); } - static IncompleteKey fromPb(DatastoreV1.Key keyPb) { + static PartialKey fromPb(DatastoreV1.Key keyPb) { String dataset = null; String namespace = null; if (keyPb.hasPartitionId()) { @@ -273,7 +273,7 @@ static IncompleteKey fromPb(DatastoreV1.Key keyPb) { for (DatastoreV1.Key.PathElement pathElementPb : keyPb.getPathElementList()) { pathBuilder.add(PathElement.fromPb(pathElementPb)); } - return new IncompleteKey(dataset, namespace, pathBuilder.build()); + return new PartialKey(dataset, namespace, pathBuilder.build()); } DatastoreV1.Key toPb() { diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java new file mode 100644 index 000000000000..ba9289aa8e4e --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Transaction.java @@ -0,0 +1,8 @@ +package com.google.gcloud.datastore; + +public interface Transaction extends DatastoreReader, DatastoreWriter { + + void commit(); + + void rollback(); +} \ No newline at end of file diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/src/main/java/com/google/gcloud/datastore/TransactionOption.java new file mode 100644 index 000000000000..50bd2aac2869 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/TransactionOption.java @@ -0,0 +1,59 @@ +package com.google.gcloud.datastore; + +import java.io.Serializable; + +public abstract class TransactionOption implements Serializable { + + private static final long serialVersionUID = -1862234444015690375L; + + + TransactionOption() { + // package protected + } + + public static final class IsolationLevel extends TransactionOption { + + private static final long serialVersionUID = -5592165378565409515L; + + private final Level level; + + public enum Level { + SERIALIZABLE, SNAPSHOT; + } + + public IsolationLevel(Level level) { + this.level = level; + } + + public Level getLevel() { + return level; + } + } + + public static final class ForceWrites extends TransactionOption { + + private static final long serialVersionUID = 7448106703678852594L; + + private final boolean force; + + public ForceWrites(boolean force) { + this.force = force; + } + + public boolean isForce() { + return force; + } + } + + public static IsolationLevel serializable() { + return new IsolationLevel(IsolationLevel.Level.SERIALIZABLE); + } + + public static IsolationLevel snapshot() { + return new IsolationLevel(IsolationLevel.Level.SNAPSHOT); + } + + public static ForceWrites forceWrites() { + return new ForceWrites(true); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java index 2a685eaf5acd..4978b14bd6ec 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -30,7 +30,7 @@ public enum Type { NULL(NullValue.MARSHALLER, NullValue.MARSHALLER), STRING(StringValue.MARSHALLER, StringValue.MARSHALLER), - EMBEDDED_ENTITY(EmbeddedEntityValue.MARSHALLER, EmbeddedEntityValue.MARSHALLER), + PARTIAL_ENTITY(PartialEntityValue.MARSHALLER, PartialEntityValue.MARSHALLER), LIST(ListValue.MARSHALLER, ListValue.MARSHALLER), KEY(KeyValue.MARSHALLER, KeyValue.MARSHALLER); diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java new file mode 100644 index 000000000000..7f4f497dd970 --- /dev/null +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -0,0 +1,69 @@ +package com.google.gcloud.datastore; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class DatastoreServiceTest { + + @Test + public void testGetOptions() { + fail("Not yet implemented"); + } + + @Test + public void testNewTransaction() { + fail("Not yet implemented"); + } + + @Test + public void testNewBatchWriter() { + fail("Not yet implemented"); + } + + @Test + public void testAllocateId() { + fail("Not yet implemented"); + } + + @Test + public void testAllocateIds() { + fail("Not yet implemented"); + } + + @Test + public void testGetKey() { + fail("Not yet implemented"); + } + + @Test + public void testGetKeyArray() { + fail("Not yet implemented"); + } + + @Test + public void testAdd() { + fail("Not yet implemented"); + } + + @Test + public void testUpdate() { + fail("Not yet implemented"); + } + + @Test + public void testPut() { + fail("Not yet implemented"); + } + + @Test + public void testDelete() { + fail("Not yet implemented"); + } + + @Test + public void testNewKeyBuilder() { + fail("Not yet implemented"); + } + +} diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 48a91eceb8e4..838c0a2079e7 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -15,8 +15,8 @@ public class SerializationTest { - private static final IncompleteKey INCOMPLETE_KEY = - new IncompleteKey.Builder("ds", "k").addToPath("p", 1).build(); + private static final PartialKey INCOMPLETE_KEY = + new PartialKey.Builder("ds", "k").addToPath("p", 1).build(); private static final Key KEY1 = new Key.Builder("ds", "k", "n").build(); private static final Key KEY2 = new Key.Builder(INCOMPLETE_KEY, 2).build(); private static final KeyValue KEY_PROPERTY = new KeyValue(KEY1); @@ -26,20 +26,20 @@ public class SerializationTest { private static final Entity ENTITY1 = new Entity.Builder(KEY1).build(); private static final Entity ENTITY2 = new Entity.Builder(KEY2).setProperty("null", new NullValue()).build(); - private static final EmbeddedEntity EMBEDDED_ENTITY1 = new EmbeddedEntity(ENTITY1); - private static final EmbeddedEntity EMBEDDED_ENTITY2 = new EmbeddedEntity(ENTITY2); - private static final EmbeddedEntity EMBEDDED_ENTITY3 = - new EmbeddedEntity.Builder() + private static final PartialEntity EMBEDDED_ENTITY1 = ENTITY1; + private static final PartialEntity EMBEDDED_ENTITY2 = ENTITY2; + private static final PartialEntity EMBEDDED_ENTITY3 = + new PartialEntity.Builder() .setKey(INCOMPLETE_KEY) .setProperty("p1", STRING_PROPERTY) .setProperty("p2", STRING_PROPERTY) .build(); - private static final EmbeddedEntityValue EMBEDDED_ENTITY_PROPERTY1 = - new EmbeddedEntityValue(EMBEDDED_ENTITY1); - private static final EmbeddedEntityValue EMBEDDED_ENTITY_PROPERTY2 = - new EmbeddedEntityValue(EMBEDDED_ENTITY2); - private static final EmbeddedEntityValue EMBEDDED_ENTITY_PROPERTY3 = - new EmbeddedEntityValue(EMBEDDED_ENTITY3); + private static final PartialEntityValue EMBEDDED_ENTITY_PROPERTY1 = + new PartialEntityValue(EMBEDDED_ENTITY1); + private static final PartialEntityValue EMBEDDED_ENTITY_PROPERTY2 = + new PartialEntityValue(EMBEDDED_ENTITY2); + private static final PartialEntityValue EMBEDDED_ENTITY_PROPERTY3 = + new PartialEntityValue(EMBEDDED_ENTITY3); private static final ListValue PROPERTY_LIST_PROPERTY = new ListValue.Builder() .addValue(NULL_PROPERTY) @@ -51,7 +51,7 @@ public class SerializationTest { Type.NULL, array(NULL_PROPERTY), Type.KEY, array(KEY_PROPERTY), Type.STRING, array(STRING_PROPERTY), - Type.EMBEDDED_ENTITY, array(EMBEDDED_ENTITY_PROPERTY1, EMBEDDED_ENTITY_PROPERTY2, + Type.PARTIAL_ENTITY, array(EMBEDDED_ENTITY_PROPERTY1, EMBEDDED_ENTITY_PROPERTY2, EMBEDDED_ENTITY_PROPERTY3), Type.LIST, array(PROPERTY_LIST_PROPERTY)); From c8c83045df5330a98888e6a3b73f126f9449a859 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 2 Dec 2014 16:33:23 -0800 Subject: [PATCH 024/732] work in progress (query) --- .../com/google/gcloud/datastore/Cursor.java | 68 ++++++++++ .../gcloud/datastore/DatastoreReader.java | 8 +- .../gcloud/datastore/DatastoreService.java | 60 ++++++--- .../datastore/DatastoreServiceImpl.java | 14 ++- .../gcloud/datastore/DatastoreWriter.java | 23 +++- .../com/google/gcloud/datastore/Entity.java | 13 +- .../com/google/gcloud/datastore/GqlQuery.java | 26 ++++ .../google/gcloud/datastore/KeyBuilder.java | 34 +++-- .../gcloud/datastore/PartialEntity.java | 54 +++----- .../google/gcloud/datastore/PartialKey.java | 116 ++++++------------ .../com/google/gcloud/datastore/Query.java | 8 ++ .../google/gcloud/datastore/QueryResult.java | 36 ++++++ .../google/gcloud/datastore/Serializable.java | 48 ++++++++ .../google/gcloud/datastore/Transaction.java | 5 +- .../com/google/gcloud/datastore/Value.java | 29 ++--- .../gcloud/datastore/SerializationTest.java | 6 +- 16 files changed, 364 insertions(+), 184 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/Cursor.java create mode 100644 src/main/java/com/google/gcloud/datastore/GqlQuery.java create mode 100644 src/main/java/com/google/gcloud/datastore/Query.java create mode 100644 src/main/java/com/google/gcloud/datastore/QueryResult.java create mode 100644 src/main/java/com/google/gcloud/datastore/Serializable.java diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java new file mode 100644 index 000000000000..51f2652b64cc --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Cursor.java @@ -0,0 +1,68 @@ +package com.google.gcloud.datastore; + +import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.protobuf.ByteString; + +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.Arrays; + +public final class Cursor implements Serializable { + + private static final long serialVersionUID = -1423744878777486541L; + + private byte[] bytes; + + public Cursor(byte[] bytes) { + this.bytes = checkNotNull(bytes); + } + + @Override + public int hashCode() { + return Arrays.hashCode(bytes); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Cursor)) { + return false; + } + + return Arrays.equals(bytes, ((Cursor) obj).bytes); + } + + @Override + public String toString() { + return toPb().toString(); + } + + public String toUrlSafe() { + try { + return URLEncoder.encode(toString(), UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unxpeced encoding exception", e); + } + } + + ByteString toPb() { + return ByteString.copyFrom(bytes); + } + + public static Cursor fromUrlSafe(String urlSafe) { + try { + String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name()); + ByteString byteString = ByteString.copyFromUtf8(utf8Str); + return fromPb(byteString); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unxpeced decoding exception", e); + } + } + + private static Cursor fromPb(ByteString byteString) { + return new Cursor(byteString.toByteArray()); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java index a1eb56aab46a..1843d147165e 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java @@ -2,6 +2,9 @@ import java.util.Iterator; +/** + * An interface to represent Google Cloud Datastore read operations. + */ public interface DatastoreReader { Entity get(Key key); @@ -9,6 +12,5 @@ public interface DatastoreReader { // results are returned using request order Iterator get(Key... key); - // query result item is a tuple of (key, value...) where values may be empty - //QueryResult runQuery(Query query); -} \ No newline at end of file + QueryResult runQuery(Query query); +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 88993173bbb8..d118b9b6c71e 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -2,27 +2,20 @@ import java.util.Iterator; +/** + * An interface for Google Cloud Datastore. + */ public interface DatastoreService extends DatastoreReader, DatastoreWriter { - enum QueryType { - PROJECTION, - FULL; - } - - /* - interface QueryResult { - - } - - // consider 2 types of queries regualr and Gql - interface Query { - QueryResult keysOnly(QueryType query); - } -*/ - - + /** + * Returns the {@code DatastoreServiceOptions} for this service. + */ DatastoreServiceOptions getOptions(); + /** + * Returns a key builder for the requested {@code kind}. + * The key would be initialized with the this service dataset and default namespace. + */ KeyBuilder newKeyBuilder(String kind); /** @@ -32,6 +25,10 @@ interface Query { */ Transaction newTransaction(TransactionOption... transactionOption); + /** + * Returns a new Batch writer for processing multiple write operations + * in one request. + */ BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption); /** @@ -46,6 +43,33 @@ interface Query { * * @throws DatastoreServiceExcepiton upon failure */ - // results are returned using request order Iterator allocateIds(PartialKey... key); + + /** + * @see DatastoreWriter#add(Entity...) + * @throws DatastoreServiceExcepiton upon failure + */ + @Override + void add(Entity... entity); + + /** + * @see DatastoreWriter#update(Entity...) + * @throws DatastoreServiceExcepiton upon failure + */ + @Override + void update(Entity... entity); + + /** + * @see DatastoreWriter#put(Entity...) + * @throws DatastoreServiceExcepiton upon failure + */ + @Override + void put(Entity... entity); + + /** + * @see DatastoreWriter#delete(Key...) + * @throws DatastoreServiceExcepiton upon failure + */ + @Override + void delete(Key... key); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index cd5475c12bab..dd6f957b92f9 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -55,29 +55,31 @@ public Iterator get(Key... key) { @Override public void add(Entity... entity) { // TODO Auto-generated method stub - } @Override public void update(Entity... entity) { // TODO Auto-generated method stub - } @Override - public Key put(Entity... entity) { + public void put(Entity... entity) { // TODO Auto-generated method stub - return null; } @Override public void delete(Key... key) { // TODO Auto-generated method stub - } @Override public KeyBuilder newKeyBuilder(String kind) { - return new KeyBuilder(kind, options.getDefaultNamespace(), kind); + return new KeyBuilder(this, kind); + } + + @Override + public QueryResult runQuery(Query query) { + // TODO Auto-generated method stub + return null; } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java index 005df9ff6800..cfb864744099 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java @@ -1,12 +1,31 @@ package com.google.gcloud.datastore; +/** + * An interface to represent Google Cloud Datastore write operations. + */ public interface DatastoreWriter { + /** + * A Datastore add operation. + * The operation will fail if an entity already exists. + */ void add(Entity... entity); + /** + * A Datastore update operation. + * The operation will fail if an entity does not already exist. + */ void update(Entity... entity); - Key put(Entity... entity); + /** + * A Datastore put (a.k.a upsert) operation. + * The operation will add or modify the entities. + */ + void put(Entity... entity); + /** + * A datastore delete operation. + * It is OK request a deletion of a non-existing entity. + */ void delete(Key... key); -} \ No newline at end of file +} diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index 7673015bb227..b1241a16ed16 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -5,8 +5,7 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSortedMap; - -import java.io.ObjectStreamException; +import com.google.protobuf.InvalidProtocolBufferException; public final class Entity extends PartialEntity { @@ -58,6 +57,11 @@ public Key getKey() { return (Key) super.getKey(); } + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Entity.parseFrom(bytesPb)); + } + static Entity fromPb(DatastoreV1.Entity entityPb) { Preconditions.checkArgument(entityPb.hasKey()); Builder builder = new Builder(Key.fromPb(entityPb.getKey())); @@ -66,9 +70,4 @@ static Entity fromPb(DatastoreV1.Entity entityPb) { } return builder.build(); } - - @Override - protected Object readResolve() throws ObjectStreamException { - return fromPb(tempEntityPb); - } } diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java new file mode 100644 index 000000000000..b99ca22ad226 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java @@ -0,0 +1,26 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.protobuf.InvalidProtocolBufferException; + +public class GqlQuery extends Query { + + private static final long serialVersionUID = 5988280590929540569L; + + @Override + protected DatastoreV1.GqlQuery toPb() { + // TODO Auto-generated method stub + return null; + } + + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + // TODO Auto-generated method stub + return null; + } + + static GqlQuery fromPb(DatastoreV1.GqlQuery queryPb) { + // TODO Auto-generated method stub + return null; + } +} diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java index dcc5fecad742..78f6fc1790de 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -1,14 +1,22 @@ package com.google.gcloud.datastore; +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * An helper for creating keys for a {@link DatastoreService}. + */ public final class KeyBuilder { private final PartialKey.Builder delegate; - - public KeyBuilder(String dataset, String namespace, String kind) { - delegate = new PartialKey.Builder(dataset, kind); - if (namespace != null) { - delegate.setNamespace(namespace); - } + private final DatastoreService service; + + /** + * Constructing a KeyBuilder. + */ + public KeyBuilder(DatastoreService service, String kind) { + this.service = checkNotNull(service); + delegate = new PartialKey.Builder(service.getOptions().getDataset(), kind); + delegate.setNamespace(service.getOptions().getDefaultNamespace()); } public KeyBuilder addToPath(String kind, String name) { @@ -26,8 +34,12 @@ public KeyBuilder setNamespace(String namespace) { return this; } - public PartialKey build() { - return delegate.build(); + /** + * Builds a key with a newly allocated id. + * @throws DatastoreServiceException if allocation failed. + */ + public Key allocateIdAndBuild() { + return service.allocateId(build()); } public Key build(String name) { @@ -37,4 +49,8 @@ public Key build(String name) { public Key build(long id) { return new Key.Builder(build(), id).build(); } -} \ No newline at end of file + + public PartialKey build() { + return delegate.build(); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index b96e4ad7b6e9..d97e168c27c6 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -2,24 +2,19 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.common.collect.ImmutableSortedMap; +import com.google.protobuf.InvalidProtocolBufferException; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.ObjectStreamException; -import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; -public class PartialEntity implements Serializable { +public class PartialEntity extends Serializable { private static final long serialVersionUID = 6492561268709192891L; private final transient PartialKey key; private final transient ImmutableSortedMap> properties; - protected transient DatastoreV1.Entity tempEntityPb; // only for deserialization public static final class Builder { @@ -84,9 +79,8 @@ public Set getPropertyNames() { return properties.keySet(); } - @Override - public String toString() { - return toPb().toString(); + ImmutableSortedMap> getProperties() { + return properties; } @Override @@ -104,21 +98,7 @@ public boolean equals(Object obj) { && Objects.equals(properties, other.properties); } - ImmutableSortedMap> getProperties() { - return properties; - } - - static PartialEntity fromPb(DatastoreV1.Entity entityPb) { - Builder builder = new Builder(); - if (entityPb.hasKey()) { - builder.setKey(PartialKey.fromPb(entityPb.getKey())); - } - for (DatastoreV1.Property property : entityPb.getPropertyList()) { - builder.setProperty(property.getName(), Value.fromPb(property.getValue())); - } - return builder.build(); - } - + @Override protected DatastoreV1.Entity toPb() { DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); if (key != null) { @@ -133,19 +113,19 @@ protected DatastoreV1.Entity toPb() { return entityPb.build(); } - private void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - out.writeObject(toPb().toByteArray()); - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - in.defaultReadObject(); - byte[] bytes = (byte[]) in.readObject(); - tempEntityPb = DatastoreV1.Entity.parseFrom(bytes); + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Entity.parseFrom(bytesPb)); } - @SuppressWarnings("unused") - protected Object readResolve() throws ObjectStreamException { - return fromPb(tempEntityPb); + static PartialEntity fromPb(DatastoreV1.Entity entityPb) { + Builder builder = new Builder(); + if (entityPb.hasKey()) { + builder.setKey(PartialKey.fromPb(entityPb.getKey())); + } + for (DatastoreV1.Property property : entityPb.getPropertyList()) { + builder.setProperty(property.getName(), Value.fromPb(property.getValue())); + } + return builder.build(); } } diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 29856f7e11c0..261e8b2b3f6b 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -9,33 +9,27 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.protobuf.InvalidProtocolBufferException; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.ObjectStreamException; -import java.io.Serializable; import java.util.LinkedList; import java.util.List; import java.util.Objects; -public class PartialKey implements Serializable { +public class PartialKey extends Serializable { private static final long serialVersionUID = -75301206578793347L; private final transient String dataset; private final transient String namespace; private final transient ImmutableList path; - private transient DatastoreV1.Key tempKeyPb; // only for deserialization - public static final class PathElement implements Serializable { + public static final class PathElement extends Serializable { private static final long serialVersionUID = -7968078857690784595L; private final transient String kind; private final transient Long id; private final transient String name; - private transient DatastoreV1.Key.PathElement tempPathElementPb; // only for deserialization private PathElement(String kind) { this(kind, null); @@ -77,11 +71,6 @@ public Object getNameOrId() { return id == null ? name : id; } - @Override - public String toString() { - return toPb().toString(); - } - @Override public int hashCode() { return Objects.hash(kind, id, name); @@ -98,17 +87,8 @@ public boolean equals(Object obj) { && Objects.equals(name, other.name); } - static PathElement fromPb(DatastoreV1.Key.PathElement pathElementPb) { - String kind = pathElementPb.getKind(); - if (pathElementPb.hasId()) { - return new PathElement(kind, pathElementPb.getId()); - } else if (pathElementPb.hasName()) { - return new PathElement(kind, pathElementPb.getName()); - } - return new PathElement(kind); - } - - DatastoreV1.Key.PathElement toPb() { + @Override + protected DatastoreV1.Key.PathElement toPb() { DatastoreV1.Key.PathElement.Builder pathElementPb = DatastoreV1.Key.PathElement.newBuilder(); pathElementPb.setKind(kind); if (id != null) { @@ -119,20 +99,19 @@ DatastoreV1.Key.PathElement toPb() { return pathElementPb.build(); } - private void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - out.writeObject(toPb().toByteArray()); - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - in.defaultReadObject(); - byte[] bytes = (byte[]) in.readObject(); - tempPathElementPb = DatastoreV1.Key.PathElement.parseFrom(bytes); + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Key.PathElement.parseFrom(bytesPb)); } - @SuppressWarnings("unused") - private Object readResolve() throws ObjectStreamException { - return fromPb(tempPathElementPb); + static PathElement fromPb(DatastoreV1.Key.PathElement pathElementPb) { + String kind = pathElementPb.getKind(); + if (pathElementPb.hasId()) { + return new PathElement(kind, pathElementPb.getId()); + } else if (pathElementPb.hasName()) { + return new PathElement(kind, pathElementPb.getName()); + } + return new PathElement(kind); } } @@ -232,11 +211,6 @@ public String getKind() { return getLeaf().getKind(); } - @Override - public String toString() { - return toPb().toString(); - } - @Override public int hashCode() { return Objects.hash(dataset, namespace, path); @@ -253,30 +227,8 @@ public boolean equals(Object other) { && Objects.equals(path, otherKey.path); } - PathElement getLeaf() { - return path.get(path.size() - 1); - } - - static PartialKey fromPb(DatastoreV1.Key keyPb) { - String dataset = null; - String namespace = null; - if (keyPb.hasPartitionId()) { - DatastoreV1.PartitionId partitionIdPb = keyPb.getPartitionId(); - if (partitionIdPb.hasDatasetId()) { - dataset = partitionIdPb.getDatasetId(); - } - if (partitionIdPb.hasNamespace()) { - namespace = partitionIdPb.getNamespace(); - } - } - ImmutableList.Builder pathBuilder = ImmutableList.builder(); - for (DatastoreV1.Key.PathElement pathElementPb : keyPb.getPathElementList()) { - pathBuilder.add(PathElement.fromPb(pathElementPb)); - } - return new PartialKey(dataset, namespace, pathBuilder.build()); - } - - DatastoreV1.Key toPb() { + @Override + protected DatastoreV1.Key toPb() { DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder(); if (dataset != null) { @@ -294,19 +246,31 @@ DatastoreV1.Key toPb() { return keyPb.build(); } - private void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - out.writeObject(toPb().toByteArray()); + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Key.parseFrom(bytesPb)); } - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - in.defaultReadObject(); - byte[] bytes = (byte[]) in.readObject(); - tempKeyPb = DatastoreV1.Key.parseFrom(bytes); + static PartialKey fromPb(DatastoreV1.Key keyPb) { + String dataset = null; + String namespace = null; + if (keyPb.hasPartitionId()) { + DatastoreV1.PartitionId partitionIdPb = keyPb.getPartitionId(); + if (partitionIdPb.hasDatasetId()) { + dataset = partitionIdPb.getDatasetId(); + } + if (partitionIdPb.hasNamespace()) { + namespace = partitionIdPb.getNamespace(); + } + } + ImmutableList.Builder pathBuilder = ImmutableList.builder(); + for (DatastoreV1.Key.PathElement pathElementPb : keyPb.getPathElementList()) { + pathBuilder.add(PathElement.fromPb(pathElementPb)); + } + return new PartialKey(dataset, namespace, pathBuilder.build()); } - @SuppressWarnings("unused") - protected Object readResolve() throws ObjectStreamException { - return fromPb(tempKeyPb); + PathElement getLeaf() { + return path.get(path.size() - 1); } } diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java new file mode 100644 index 000000000000..cff0b13aaf73 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Query.java @@ -0,0 +1,8 @@ +package com.google.gcloud.datastore; + +import com.google.protobuf.GeneratedMessage; + +public abstract class Query extends Serializable { + + private static final long serialVersionUID = 4960655852209261775L; +} diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java new file mode 100644 index 000000000000..d47edc484ddd --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java @@ -0,0 +1,36 @@ +package com.google.gcloud.datastore; + +import java.util.Iterator; + +/** + * A result of Google Cloud Datastore query. + */ +public interface QueryResult extends Iterator { + + /** + * The type of the result. + * Full: A complete {@link Entity}. + * PROJECTION: A partial entity, represented by {@link PartialEntity}. + * KEY_ONLY: An entity's {@link Key}. + */ + enum ResultType { + FULL, + PROJECTION, + KEY_ONLY; + } + + /** + * This method should be used to cast the result to its appropriate type. + * 

 {@code
+   * ResultType.FULL -> (QueryResult)
+   * ResultType.PROJECTION -> (QueryResult)
+   * ResultType.KEY_ONLY -> (QueryResult)
+   * }
+ */ + ResultType getResultType(); + + /** + * Return the Cursor for the next result. Not currently implemented (depends on v1beta3). + */ + Cursor getCursor(); +} diff --git a/src/main/java/com/google/gcloud/datastore/Serializable.java b/src/main/java/com/google/gcloud/datastore/Serializable.java new file mode 100644 index 000000000000..c9f6b7984512 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Serializable.java @@ -0,0 +1,48 @@ +package com.google.gcloud.datastore; + +import com.google.protobuf.GeneratedMessage; +import com.google.protobuf.InvalidProtocolBufferException; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamException; +import java.io.StreamCorruptedException; + +abstract class Serializable implements java.io.Serializable { + + private static final long serialVersionUID = -5565522710061949199L; + + private transient byte[] bytesPb; // only for deserialization + + @Override + public String toString() { + return toPb().toString(); + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + out.writeObject(toPb().toByteArray()); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + bytesPb = (byte[]) in.readObject(); + } + + protected Object readResolve() throws ObjectStreamException { + try { + return fromPb(bytesPb); + } catch (InvalidProtocolBufferException ex) { + StreamCorruptedException sce = new StreamCorruptedException("Failed to create object"); + sce.initCause(ex); + throw sce; + } finally { + bytesPb = null; + } + } + + protected abstract M toPb(); + + protected abstract Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException; +} diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java index ba9289aa8e4e..4df12134c3ed 100644 --- a/src/main/java/com/google/gcloud/datastore/Transaction.java +++ b/src/main/java/com/google/gcloud/datastore/Transaction.java @@ -1,8 +1,11 @@ package com.google.gcloud.datastore; +/** + * A Google cloud datastore transaction. + */ public interface Transaction extends DatastoreReader, DatastoreWriter { void commit(); void rollback(); -} \ No newline at end of file +} diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java index 4978b14bd6ec..edc197ddd696 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -5,18 +5,14 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.InvalidProtocolBufferException; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.ObjectStreamException; -import java.io.Serializable; import java.util.Objects; // TODO: add javadoc, and mention that null should only be represented by NullValue. public abstract class Value, B extends Value.Builder> - implements Serializable { + extends Serializable { private static final long serialVersionUID = -1899638277588872742L; @@ -24,7 +20,6 @@ private final transient boolean indexed; private final transient Integer meaning; private final transient V value; - private transient DatastoreV1.Value tempValuePb; // only for deserialization public enum Type { @@ -262,8 +257,9 @@ public boolean equals(Object other) { && Objects.equals(value, otherValue.get()); } + @Override @SuppressWarnings("unchecked") - DatastoreV1.Value toPb() { + protected DatastoreV1.Value toPb() { Marshaller marshaller = getType().getMarshaller(); return marshaller.toProto((P) this); } @@ -293,19 +289,8 @@ DatastoreV1.Value toPb() { return marshaller.fromProto(proto).build(); } - private void writeObject(ObjectOutputStream out) throws IOException { - out.defaultWriteObject(); - out.writeObject(toPb().toByteArray()); - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - in.defaultReadObject(); - byte[] bytes = (byte[]) in.readObject(); - tempValuePb = DatastoreV1.Value.parseFrom(bytes); - } - - @SuppressWarnings("unused") - protected Object readResolve() throws ObjectStreamException { - return fromPb(tempValuePb); + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Value.parseFrom(bytesPb)); } } diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 838c0a2079e7..0184f26717b9 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -62,9 +62,9 @@ private static T[] array(T... values) { @Test public void testValues() throws Exception { for (Type type : Type.values()) { - Value[] values = typeToValues.get(type); - for (Value value : values) { - Value copy = serialiazeAndDeserialize(value); + Value[] values = typeToValues.get(type); + for (Value value : values) { + Value copy = serialiazeAndDeserialize(value); assertEquals(value, copy); assertEquals(value.get(), copy.get()); } From ab71adf45cffda1dd40c193dd50b2bf59473c266 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 3 Dec 2014 18:41:55 -0800 Subject: [PATCH 025/732] added all types --- pom.xml | 5 + .../google/gcloud/datastore/BatchWriter.java | 7 +- .../com/google/gcloud/datastore/Blob.java | 128 ++++++++++++ .../google/gcloud/datastore/BlobValue.java | 55 +++++ .../google/gcloud/datastore/BooleanValue.java | 55 +++++ .../com/google/gcloud/datastore/Cursor.java | 12 +- .../gcloud/datastore/DatastoreReader.java | 17 +- .../gcloud/datastore/DatastoreService.java | 2 +- .../datastore/DatastoreServiceImpl.java | 7 +- .../google/gcloud/datastore/DateAndTime.java | 74 +++++++ .../gcloud/datastore/DateAndTimeValue.java | 57 +++++ .../google/gcloud/datastore/DoubleValue.java | 55 +++++ .../com/google/gcloud/datastore/Entity.java | 43 ++-- .../com/google/gcloud/datastore/GqlQuery.java | 2 +- .../java/com/google/gcloud/datastore/Key.java | 140 +++++++++---- .../google/gcloud/datastore/KeyBuilder.java | 8 +- .../com/google/gcloud/datastore/KeyValue.java | 4 +- .../google/gcloud/datastore/ListValue.java | 7 +- .../google/gcloud/datastore/LongValue.java | 55 +++++ .../gcloud/datastore/PartialEntity.java | 55 +++-- .../gcloud/datastore/PartialEntityValue.java | 4 +- .../google/gcloud/datastore/PartialKey.java | 73 ++++--- .../com/google/gcloud/datastore/Query.java | 5 +- .../google/gcloud/datastore/QueryResult.java | 39 ++-- .../com/google/gcloud/datastore/RawValue.java | 59 ++++++ .../google/gcloud/datastore/Serializable.java | 17 ++ .../google/gcloud/datastore/StringValue.java | 6 +- .../google/gcloud/datastore/Transaction.java | 10 + .../com/google/gcloud/datastore/Value.java | 196 ++++++++++++------ .../datastore/DatastoreServiceTest.java | 3 +- .../gcloud/datastore/SerializationTest.java | 106 ++++++---- 31 files changed, 1051 insertions(+), 255 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/Blob.java create mode 100644 src/main/java/com/google/gcloud/datastore/BlobValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/BooleanValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/DateAndTime.java create mode 100644 src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/DoubleValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/LongValue.java create mode 100644 src/main/java/com/google/gcloud/datastore/RawValue.java diff --git a/pom.xml b/pom.xml index d82863493cd8..478c89fd38d0 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,11 @@ test RELEASE + + joda-time + joda-time + RELEASE + diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java index 9d2e74f01cb1..c8e1056b21d9 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java @@ -2,5 +2,10 @@ public interface BatchWriter extends DatastoreWriter { + /** + * Submit the batch to the Datastore. + * + * @throws DatastoreServiceException if there was any failure. + */ void submit(); -} \ No newline at end of file +} diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java new file mode 100644 index 000000000000..e7e134815236 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/Blob.java @@ -0,0 +1,128 @@ +package com.google.gcloud.datastore; + +import static com.google.api.client.util.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.protobuf.ByteString; + +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * A Google Cloud Datastore Blob. + * A Datastore blob is limited to {@value #MAX_LENGTH} bytes. + * This class is immutable. + * + * @see Google Cloud Datastore Entities, Properties, and Keys + */ +public final class Blob implements java.io.Serializable { + + private static final long serialVersionUID = 3835421019618247721L; + private static final int MAX_LENGTH = 1_000_000; + + private final ByteString byteString; + + Blob(ByteString byteString, boolean enforceLimits) { + this.byteString = checkNotNull(byteString); + if (enforceLimits) { + checkArgument(byteString.size() <= MAX_LENGTH, "May be a maximum of 1,000,000 bytes"); + } + } + + @Override + public String toString() { + return byteString.toString(); + } + + @Override + public int hashCode() { + return byteString.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Blob)) { + return false; + } + return Objects.equals(byteString, ((Blob) obj).byteString); + } + + /** + * Returns the size of this blob. + */ + public int length() { + return byteString.size(); + } + + /** + * Returns a copy as byte array. + */ + public byte[] toByteArray() { + return byteString.toByteArray(); + } + + /** + * Returns a read-only {@link ByteBuffer} for this blob content. + */ + public ByteBuffer asReadOnlyByteBuffer() { + return byteString.asReadOnlyByteBuffer(); + } + + /** + * Returns an {@link InputStream} for this blob content. + */ + public InputStream asInputStream() { + final ByteBuffer byteBuffer = asReadOnlyByteBuffer(); + return new InputStream() { + @Override public int read() throws IOException { + return !byteBuffer.hasRemaining() ? -1 : byteBuffer.get() & 0xFF; + } + }; + } + + /** + * Copies bytes into a ByteBuffer. + * + * @throws java.nio.ReadOnlyBufferException if the target is read-only + * @throws java.nio.BufferOverflowException if the target's remaining() space is not large + * enough to hold the data. + */ + public void copyTo(ByteBuffer target) { + byteString.copyTo(target); + } + + /** + * Copies bytes into a buffer. + * + * @throws java.io.IndexOutOfBoundsException if an offset or size is negative or too large + */ + public void copyTo(byte[] target) { + byteString.copyTo(target, 0, 0, length()); + } + + ByteString byteString() { + return byteString; + } + + public static Blob copyFrom(byte[] bytes) { + return new Blob(ByteString.copyFrom(bytes), true); + } + + public static Blob copyFrom(ByteBuffer bytes) { + return new Blob(ByteString.copyFrom(bytes), true); + } + + public static Blob copyFrom(InputStream input) throws IOException { + BufferedInputStream bufferedInput = new BufferedInputStream(input); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + int value; + while ((value = bufferedInput.read()) != -1) { + bytes.write(value); + } + return copyFrom(bytes.toByteArray()); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java new file mode 100644 index 000000000000..d44d384909a6 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java @@ -0,0 +1,55 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.BLOB_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public final class BlobValue extends Value { + + private static final long serialVersionUID = -5096238337676649540L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return BLOB_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(Blob value) { + return new Builder(value); + } + + @Override + protected Blob getValue(DatastoreV1.Value from) { + return new Blob(from.getBlobValue(), false); + } + + @Override + protected void setValue(BlobValue from, DatastoreV1.Value.Builder to) { + to.setBlobValue(from.get().byteString()); + } + }; + + public static final class Builder extends Value.BaseBuilder { + + public Builder(Blob blob) { + super(Type.BLOB); + set(blob); + } + + @Override + public BlobValue build() { + return new BlobValue(this); + } + } + + public BlobValue(Blob blob) { + this(new Builder(blob)); + } + + BlobValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/src/main/java/com/google/gcloud/datastore/BooleanValue.java new file mode 100644 index 000000000000..6eebdbf69b56 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/BooleanValue.java @@ -0,0 +1,55 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.BOOLEAN_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public final class BooleanValue extends Value { + + private static final long serialVersionUID = -542649497897250340L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return BOOLEAN_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(Boolean value) { + return new Builder(value); + } + + @Override + protected Boolean getValue(DatastoreV1.Value from) { + return from.getBooleanValue(); + } + + @Override + protected void setValue(BooleanValue from, DatastoreV1.Value.Builder to) { + to.setBooleanValue(from.get()); + } + }; + + public static final class Builder extends Value.BaseBuilder { + + public Builder(boolean value) { + super(Type.BOOLEAN); + set(value); + } + + @Override + public BooleanValue build() { + return new BooleanValue(this); + } + } + + public BooleanValue(boolean value) { + this(new Builder(value)); + } + + BooleanValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java index 51f2652b64cc..e1163da38150 100644 --- a/src/main/java/com/google/gcloud/datastore/Cursor.java +++ b/src/main/java/com/google/gcloud/datastore/Cursor.java @@ -11,11 +11,15 @@ import java.net.URLEncoder; import java.util.Arrays; +/** + * A Google Cloud Datastore cursor. + * The cursor can be used to as a starting point or an ending point for a {@link Query} + */ public final class Cursor implements Serializable { private static final long serialVersionUID = -1423744878777486541L; - private byte[] bytes; + private final byte[] bytes; public Cursor(byte[] bytes) { this.bytes = checkNotNull(bytes); @@ -40,6 +44,9 @@ public String toString() { return toPb().toString(); } + /** + * Returns the cursor in an encoded form that can be used as part of a URL. + */ public String toUrlSafe() { try { return URLEncoder.encode(toString(), UTF_8.name()); @@ -52,6 +59,9 @@ ByteString toPb() { return ByteString.copyFrom(bytes); } + /** + * Create a {@code Cursor} given its URL safe encoded form. + */ public static Cursor fromUrlSafe(String urlSafe) { try { String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name()); diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java index 1843d147165e..32b4dbd73461 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java @@ -7,10 +7,25 @@ */ public interface DatastoreReader { + /** + * Returns an {@link Entity} for the given {@link Key} or {@code null} if does not exists. + * + * @throws DatastoreServiceException upon failure. + */ Entity get(Key key); - // results are returned using request order + /** + * Returns an {@link Entity} for each given {@link Key} or {@code null} if does not exists + * ordered by input. + * + * @throws DatastoreServiceException upon failure. + */ Iterator get(Key... key); + /** + * Submit a {@link Query} and returns its result. + * + * @throws DatastoreServiceException upon failure. + */ QueryResult runQuery(Query query); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index d118b9b6c71e..1b67fbe447eb 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -3,7 +3,7 @@ import java.util.Iterator; /** - * An interface for Google Cloud Datastore. + * An interface for Google Cloud Datastore dataset. */ public interface DatastoreService extends DatastoreReader, DatastoreWriter { diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index dd6f957b92f9..fb73952613dc 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -30,20 +30,19 @@ public BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption) { @Override public Key allocateId(PartialKey key) { - // TODO Auto-generated method stub - return null; + return allocateIds(key).next(); } @Override public Iterator allocateIds(PartialKey... key) { // TODO Auto-generated method stub + // Will need to populate "force" after b/18594027 is fixed. return null; } @Override public Entity get(Key key) { - // TODO Auto-generated method stub - return null; + return get(new Key[]{key}).next(); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/DateAndTime.java b/src/main/java/com/google/gcloud/datastore/DateAndTime.java new file mode 100644 index 000000000000..492802e3b8dc --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DateAndTime.java @@ -0,0 +1,74 @@ +package com.google.gcloud.datastore; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.joda.time.format.ISODateTimeFormat; + +import java.util.Calendar; +import java.util.Date; + +/** + * A Google Cloud Datastore timestamp. + * A Datastore timestamp is represented in micro-seconds. + * This class is immutable. + * + * @see Google Cloud Datastore Entities, Properties, and Keys + */ +public final class DateAndTime implements java.io.Serializable { + + private static final long serialVersionUID = 7343324797621228378L; + + private final long timestampMicroseconds; + + DateAndTime(long timestampMicroseconds) { + this.timestampMicroseconds = timestampMicroseconds; + } + + @Override + public String toString() { + return ISODateTimeFormat.dateTime().print(timestampMillis()); + } + + @Override + public int hashCode() { + return (int) timestampMicroseconds; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof DateAndTime)) { + return false; + } + return timestampMicroseconds == ((DateAndTime) obj).timestampMicroseconds; + } + + public long timestampMicroseconds() { + return timestampMicroseconds; + } + + public long timestampMillis() { + return timestampMicroseconds / 1000L; + } + + public Date toDate() { + return new Date(timestampMillis()); + } + + public Calendar toCalendar() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(timestampMillis()); + return cal; + } + + public static DateAndTime now() { + return new DateAndTime(System.nanoTime() / 1000L); + } + + public static DateAndTime copyFrom(Date date) { + return new DateAndTime(checkNotNull(date).getTime() * 1000L); + } + + public static DateAndTime copyFrom(Calendar calendar) { + return copyFrom(calendar.getTime()); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java new file mode 100644 index 000000000000..e1fa416c6bb1 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java @@ -0,0 +1,57 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public final class DateAndTimeValue + extends Value { + + private static final long serialVersionUID = -5096238337676649540L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(DateAndTime value) { + return new Builder(value); + } + + @Override + protected DateAndTime getValue(DatastoreV1.Value from) { + return new DateAndTime(from.getTimestampMicrosecondsValue()); + } + + @Override + protected void setValue(DateAndTimeValue from, DatastoreV1.Value.Builder to) { + to.setTimestampMicrosecondsValue(from.get().timestampMicroseconds()); + } + }; + + public static final class Builder + extends Value.BaseBuilder { + + public Builder(DateAndTime dateAndTime) { + super(Type.DATE_AND_TIME); + set(dateAndTime); + } + + @Override + public DateAndTimeValue build() { + return new DateAndTimeValue(this); + } + } + + public DateAndTimeValue(DateAndTime dateAndTime) { + this(new Builder(dateAndTime)); + } + + DateAndTimeValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/src/main/java/com/google/gcloud/datastore/DoubleValue.java new file mode 100644 index 000000000000..dca443a32851 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DoubleValue.java @@ -0,0 +1,55 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.DOUBLE_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public final class DoubleValue extends Value { + + private static final long serialVersionUID = -5096238337676649540L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return DOUBLE_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(Double value) { + return new Builder(value); + } + + @Override + protected Double getValue(DatastoreV1.Value from) { + return from.getDoubleValue(); + } + + @Override + protected void setValue(DoubleValue from, DatastoreV1.Value.Builder to) { + to.setDoubleValue(from.get()); + } + }; + + public static final class Builder extends Value.BaseBuilder { + + public Builder(double value) { + super(Type.DOUBLE); + set(value); + } + + @Override + public DoubleValue build() { + return new DoubleValue(this); + } + } + + public DoubleValue(double value) { + this(new Builder(value)); + } + + DoubleValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index b1241a16ed16..2bdd2dbfe41b 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -1,47 +1,55 @@ package com.google.gcloud.datastore; -import static com.google.common.base.Preconditions.checkNotNull; - import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSortedMap; import com.google.protobuf.InvalidProtocolBufferException; +/** + * An entity is the Google Cloud Datastore persistent data object. + * An entity holds one or more properties, represented by a name (as {@link String}) + * and a value (as {@link Value}), and is associated with a {@link Key}. + * For a list of possible values see {@link Value.Type}. + * This class is immutable. To edit (a copy) use {@link #builder()}. + * + * @see Google Cloud Datastore Entities, Properties, and Keys + */ public final class Entity extends PartialEntity { private static final long serialVersionUID = 432961565733066915L; - public static final class Builder { - - private PartialEntity.Builder delegate; + public static final class Builder extends PartialEntity.Builder { public Builder(Key key) { - delegate = new PartialEntity.Builder(); - delegate.setKey(checkNotNull(key)); + super(key); } public Builder(Entity entity) { - delegate = new PartialEntity.Builder(entity); + super(entity); } + @Override public Builder clearProperties() { - delegate.clearProperties(); + super.clearProperties(); return this; } + @Override public Builder removeProperty(String name) { - delegate.removeProperty(name); + super.removeProperty(name); return this; } + @Override public Builder setProperty(String name, Value value) { - delegate.setProperty(name, value); + super.setProperty(name, value); return this; } + @Override public Entity build() { - PartialEntity entity = delegate.build(); - return new Entity((Key) entity.getKey(), entity.getProperties()); + PartialEntity entity = super.build(); + return new Entity((Key) entity.key(), entity.properties()); } } @@ -53,8 +61,13 @@ private Entity(Key key, ImmutableSortedMap> properties) { * Returns the entity's key (never null). */ @Override - public Key getKey() { - return (Key) super.getKey(); + public Key key() { + return (Key) super.key(); + } + + @Override + public Builder builder() { + return new Builder(this); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java index b99ca22ad226..2f8f87ba0f18 100644 --- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java +++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java @@ -3,7 +3,7 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.protobuf.InvalidProtocolBufferException; -public class GqlQuery extends Query { +public class GqlQuery extends Serializable implements Query { private static final long serialVersionUID = 5988280590929540569L; diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index bc145a157b7b..33f013e8ad3f 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -1,6 +1,5 @@ package com.google.gcloud.datastore; -import static com.google.common.base.Preconditions.checkNotNull; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.datastore.DatastoreV1; @@ -8,108 +7,154 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import java.io.ObjectStreamException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; /** - * A key that is guaranteed to be complete. + * A key that is guaranteed to be complete and could be used to reference a + * Google Cloud Datastore {@link Entity}. + * This class is immutable. To edit (a copy) use {@link #builder()}. + * + * @see Google Cloud Datastore Entities, Properties, and Keys */ public final class Key extends PartialKey { private static final long serialVersionUID = 3160994559785491356L; - public static class Builder { + public static final class Builder extends PartialKey.Builder { - private final PartialKey.Builder delegate; private String name; private Long id; public Builder(String dataset, String kind, String name) { - delegate = new PartialKey.Builder(dataset, kind); + super(dataset, kind); this.name = name; } public Builder(String dataset, String kind, long id) { - delegate = new PartialKey.Builder(dataset, kind); + super(dataset, kind); this.id = id; } public Builder(PartialKey key, String name) { - delegate = new PartialKey.Builder(key); + super(key); this.name = name; } public Builder(PartialKey key, long id) { - delegate = new PartialKey.Builder(key); + super(key); this.id = id; } - public Builder setDataset(String dataset) { - delegate.setDataset(checkNotNull(dataset)); + public Builder(Key key) { + super(key); + if (key.id() == null) { + name = key.name(); + } else { + id = key.id(); + } + } + + @Override + public Builder addToPath(String kind, long id) { + super.addToPath(kind, id); return this; } - public Builder setNamespace(String namespace) { - delegate.setNamespace(checkNotNull(namespace)); + @Override + public Builder addToPath(String kind, String name) { + super.addToPath(kind, name); return this; } - public Builder setName(String name) { - this.name = name; - this.id = null; + @Override + public Builder addToPath(PathElement pathElement) { + super.addToPath(pathElement); return this; } - public Builder setId(long id) { - this.id = id; - this.name = null; + @Override + public Builder kind(String kind) { + super.kind(kind); return this; } - public Builder addToPath(String kind, long id) { - delegate.addToPath(kind, id); + @Override + public Builder clearPath() { + super.clearPath(); return this; } - public Builder addToPath(String kind, String name) { - delegate.addToPath(kind, name); + @Override + public Builder dataset(String dataset) { + super.dataset(dataset); return this; } - public Builder clearPath() { - delegate.clearPath(); + @Override + public Builder namespace(String namespace) { + super.namespace(namespace); return this; } + public Builder name(String name) { + this.name = name; + id = null; + return this; + } + + public Builder id(long id) { + this.id = id; + name = null; + return this; + } + + @Override public Key build() { - PartialKey key = delegate.build(); + PartialKey key = super.build(); return id == null ? new Key(key, name) : new Key(key, id); } } private Key(PartialKey key, String name) { - super(key.getDataset(), key.getNamespace(), newPath(key, name)); + super(key.dataset(), key.namespace(), newPath(key, name)); } private Key(PartialKey key, long id) { - super(key.getDataset(), key.getNamespace(), newPath(key, id)); + super(key.dataset(), key.namespace(), newPath(key, id)); } - public Long getId() { - return getLeaf().getId(); + @Override + public Builder builder() { + return new Builder(this); } - public String getName() { - return getLeaf().getName(); + /** + * Returns the key's id or {@code null} if it has a name instead. + */ + public Long id() { + return getLeaf().id(); } - public Object getNameOrId() { + /** + * Returns the key's name or {@code null} if it has an id instead. + */ + public String name() { + return getLeaf().name(); + } + + /** + * Returns the key's id (as {@link #Long}) or name (as {@link String}). + */ + public Object nameOrId() { PathElement leaf = getLeaf(); - return leaf.hasId() ? leaf.getId() : leaf.getName(); + return leaf.hasId() ? leaf.id() : leaf.name(); } + /** + * Returns the key in an encoded form that can be used as part of a URL. + */ public String toUrlSafe() { try { return URLEncoder.encode(toString(), UTF_8.name()); @@ -118,6 +163,11 @@ public String toUrlSafe() { } } + /** + * Create a {@code Key} given its URL safe encoded form. + * + * @throws RuntimeException when decoding fails + */ public static Key fromUrlSafe(String urlSafe) { try { String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name()); @@ -142,33 +192,33 @@ public static Key fromIncompleteKey(PartialKey key) { } PathElement leaf = key.getLeaf(); if (leaf.hasId()) { - return new Key(key, leaf.getId()); + return new Key(key, leaf.id()); } else if (leaf.hasName()) { - return new Key(key, leaf.getName()); + return new Key(key, leaf.name()); } throw new IllegalArgumentException("Key is missing name or id"); } - static Key fromPb(DatastoreV1.Key keyPb) { - return fromIncompleteKey(PartialKey.fromPb(keyPb)); + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Key.parseFrom(bytesPb)); } - @Override - protected Object readResolve() throws ObjectStreamException { - return fromIncompleteKey((PartialKey) super.readResolve()); + static Key fromPb(DatastoreV1.Key keyPb) { + return fromIncompleteKey(PartialKey.fromPb(keyPb)); } private static ImmutableList newPath(PartialKey key, String name) { ImmutableList.Builder path = ImmutableList.builder(); - path.addAll(key.getAncestorPath()); - path.add(new PathElement(key.getKind(), name)); + path.addAll(key.ancestorPath()); + path.add(new PathElement(key.kind(), name)); return path.build(); } private static ImmutableList newPath(PartialKey key, long id) { ImmutableList.Builder path = ImmutableList.builder(); - path.addAll(key.getAncestorPath()); - path.add(new PathElement(key.getKind(), id)); + path.addAll(key.ancestorPath()); + path.add(new PathElement(key.kind(), id)); return path.build(); } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java index 78f6fc1790de..07c6382f0dc5 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -3,7 +3,7 @@ import static com.google.common.base.Preconditions.checkNotNull; /** - * An helper for creating keys for a {@link DatastoreService}. + * An helper for creating keys for a specific {@link DatastoreService}/dataset. */ public final class KeyBuilder { @@ -16,7 +16,7 @@ public final class KeyBuilder { public KeyBuilder(DatastoreService service, String kind) { this.service = checkNotNull(service); delegate = new PartialKey.Builder(service.getOptions().getDataset(), kind); - delegate.setNamespace(service.getOptions().getDefaultNamespace()); + delegate.namespace(service.getOptions().getDefaultNamespace()); } public KeyBuilder addToPath(String kind, String name) { @@ -29,8 +29,8 @@ public KeyBuilder addToPath(String kind, long id) { return this; } - public KeyBuilder setNamespace(String namespace) { - delegate.setNamespace(namespace); + public KeyBuilder namespace(String namespace) { + delegate.namespace(namespace); return this; } diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java index 637fb6931bdb..671e9c355b14 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyValue.java +++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java @@ -34,9 +34,9 @@ protected void setValue(KeyValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(Key value) { + public Builder(Key key) { super(Type.KEY); - set(value); + set(key); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java index b33c096ff718..b8bdb7ec17d6 100644 --- a/src/main/java/com/google/gcloud/datastore/ListValue.java +++ b/src/main/java/com/google/gcloud/datastore/ListValue.java @@ -55,7 +55,7 @@ public Builder() { } public Builder addValue(Value value) { - Preconditions.checkArgument(value.getType() != Type.LIST, "Cannot contain another list"); + Preconditions.checkArgument(value.type() != Type.LIST, "Cannot contain another list"); listBuilder.add(value); return this; } @@ -68,6 +68,11 @@ public Builder addValue(Value first, Value... other) { return this; } + /** + * Copy the list of values. + * + * @see com.google.gcloud.datastore.Value.BaseBuilder#set(java.lang.Object) + */ @Override public Builder set(List> properties) { listBuilder = ImmutableList.>builder(); diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/src/main/java/com/google/gcloud/datastore/LongValue.java new file mode 100644 index 000000000000..9a4690a24133 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/LongValue.java @@ -0,0 +1,55 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.INTEGER_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public final class LongValue extends Value { + + private static final long serialVersionUID = -8552854340400546861L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return INTEGER_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(Long value) { + return new Builder(value); + } + + @Override + protected Long getValue(DatastoreV1.Value from) { + return from.getIntegerValue(); + } + + @Override + protected void setValue(LongValue from, DatastoreV1.Value.Builder to) { + to.setIntegerValue(from.get()); + } + }; + + public static final class Builder extends Value.BaseBuilder { + + public Builder(long value) { + super(Type.LONG); + set(value); + } + + @Override + public LongValue build() { + return new LongValue(this); + } + } + + public LongValue(long value) { + this(new Builder(value)); + } + + LongValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index d97e168c27c6..c13ea5679b64 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -9,6 +9,13 @@ import java.util.Objects; import java.util.Set; +/** + * A partial entity holds one or more properties, represented by a name (as {@link String}) + * and a value (as {@link Value}). + * For a list of possible values see {@link Value.Type}. + * A partial entity also can be associated with a key (partial or full). + * This class is immutable. To edit (a copy) use {@link #builder()}. + */ public class PartialEntity extends Serializable { private static final long serialVersionUID = 6492561268709192891L; @@ -16,23 +23,22 @@ public class PartialEntity extends Serializable { private final transient PartialKey key; private final transient ImmutableSortedMap> properties; - public static final class Builder { + public static class Builder { - private PartialKey key; - private Map> properties; + private final PartialKey key; + private final Map> properties; - public Builder() { + /** + * Construct a builder with a partial key (could be null). + */ + public Builder(PartialKey key) { + this.key = key; properties = new HashMap<>(); } public Builder(PartialEntity entity) { - this.key = entity.getKey(); - this.properties = new HashMap<>(entity.getProperties()); - } - - public Builder setKey(PartialKey key) { - this.key = key; - return this; + key = entity.key(); + properties = new HashMap<>(entity.properties()); } public Builder clearProperties() { @@ -61,9 +67,9 @@ protected PartialEntity(PartialKey key, ImmutableSortedMap getProperty(String name) { - return properties.get(name); + @SuppressWarnings("unchecked") + public , B extends Value.Builder> Value property( + String name) { + return (Value) properties.get(name); } - public Set getPropertyNames() { + public Set propertyNames() { return properties.keySet(); } - ImmutableSortedMap> getProperties() { + /** + * Returns a new builder for this entity (values are copied). + */ + public Builder builder() { + return new Builder(this); + } + + ImmutableSortedMap> properties() { return properties; } @@ -119,10 +134,8 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { } static PartialEntity fromPb(DatastoreV1.Entity entityPb) { - Builder builder = new Builder(); - if (entityPb.hasKey()) { - builder.setKey(PartialKey.fromPb(entityPb.getKey())); - } + PartialKey key = entityPb.hasKey() ? PartialKey.fromPb(entityPb.getKey()) : null; + Builder builder = new Builder(key); for (DatastoreV1.Property property : entityPb.getPropertyList()) { builder.setProperty(property.getName(), Value.fromPb(property.getValue())); } diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java b/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java index db076a3305a0..cc4dd834a343 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java @@ -5,7 +5,7 @@ import com.google.api.services.datastore.DatastoreV1; public class PartialEntityValue extends - Value { +Value { private static final long serialVersionUID = -5461475706792576395L; @@ -34,7 +34,7 @@ protected void setValue(PartialEntityValue from, DatastoreV1.Value.Builder to) { }; public static final class Builder extends - Value.BaseBuilder { + Value.BaseBuilder { public Builder(PartialEntity entity) { super(Type.PARTIAL_ENTITY); diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 261e8b2b3f6b..4bcca81ddb64 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -15,6 +15,11 @@ import java.util.List; import java.util.Objects; +/** + * A partial key (without a name or id). + * Could be used as metadata for {@link PartialEntity}. + * This class is immutable. To edit (a copy) use {@link #builder()}. + */ public class PartialKey extends Serializable { private static final long serialVersionUID = -75301206578793347L; @@ -38,16 +43,16 @@ private PathElement(String kind) { public PathElement(String kind, long id) { this.kind = kind; this.id = id; - this.name = null; + name = null; } public PathElement(String kind, String name) { this.kind = kind; this.name = name; - this.id = null; + id = null; } - public String getKind() { + public String kind() { return kind; } @@ -55,7 +60,7 @@ public boolean hasId() { return id != null; } - public long getId() { + public long id() { return id == null ? 0 : id; } @@ -63,11 +68,11 @@ public boolean hasName() { return name != null; } - public String getName() { + public String name() { return name == null ? "" : name; } - public Object getNameOrId() { + public Object nameOrId() { return id == null ? name : id; } @@ -115,12 +120,12 @@ static PathElement fromPb(DatastoreV1.Key.PathElement pathElementPb) { } } - public static final class Builder { + public static class Builder { private String dataset; private String namespace = DEFAULT_NAMESPACE; private String kind; - private List path = new LinkedList<>(); + private final List path = new LinkedList<>(); private static final String DEFAULT_NAMESPACE = ""; private static final int MAX_PATH = 100; @@ -133,8 +138,8 @@ public Builder(String dataset, String kind) { public Builder(PartialKey key) { dataset = key.dataset; namespace = key.namespace; - kind = key.getKind(); - path.addAll(key.getAncestorPath()); + kind = key.kind(); + path.addAll(key.ancestorPath()); } public Builder addToPath(String kind, long id) { @@ -148,13 +153,13 @@ public Builder addToPath(String kind, String name) { return addToPath(new PathElement(kind, name)); } - public Builder addToPath(PathElement pathEntry) { + public Builder addToPath(PathElement pathElement) { Preconditions.checkState(path.size() < MAX_PATH, "path can have at most 100 elements"); - path.add(pathEntry); + path.add(pathElement); return this; } - public Builder setKind(String kind) { + public Builder kind(String kind) { this.kind = validateKind(kind); return this; } @@ -170,12 +175,12 @@ public Builder clearPath() { return this; } - public Builder setDataset(String dataset) { + public Builder dataset(String dataset) { this.dataset = validateDataset(dataset); return this; } - public Builder setNamespace(String namespace) { + public Builder namespace(String namespace) { this.namespace = checkNotNull(namespace); return this; } @@ -195,20 +200,36 @@ public PartialKey build() { this.path = path; } - public String getDataset() { + /** + * Returns the key's dataset. + */ + public String dataset() { return dataset; } - public String getNamespace() { + /** + * Returns the key's namespace. + */ + public String namespace() { return namespace; } - public List getAncestorPath() { + /** + * Returns the key's parent's path. + */ + public List ancestorPath() { return path.subList(0, path.size() - 1); } - public String getKind() { - return getLeaf().getKind(); + /** + * Returns the key's kind. + */ + public String kind() { + return getLeaf().kind(); + } + + public Builder builder() { + return new Builder(this); } @Override @@ -217,14 +238,14 @@ public int hashCode() { } @Override - public boolean equals(Object other) { - if (!(other instanceof PartialKey)) { + public boolean equals(Object obj) { + if (!(obj instanceof PartialKey)) { return false; } - PartialKey otherKey = (PartialKey) other; - return Objects.equals(dataset, otherKey.dataset) - && Objects.equals(namespace, otherKey.namespace) - && Objects.equals(path, otherKey.path); + PartialKey other = (PartialKey) obj; + return Objects.equals(dataset, other.dataset) + && Objects.equals(namespace, other.namespace) + && Objects.equals(path, other.path); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java index cff0b13aaf73..822b7c6dc472 100644 --- a/src/main/java/com/google/gcloud/datastore/Query.java +++ b/src/main/java/com/google/gcloud/datastore/Query.java @@ -1,8 +1,7 @@ package com.google.gcloud.datastore; -import com.google.protobuf.GeneratedMessage; +import java.io.Serializable; -public abstract class Query extends Serializable { - private static final long serialVersionUID = 4960655852209261775L; +public interface Query extends Serializable { } diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java index d47edc484ddd..446fc6dc43e4 100644 --- a/src/main/java/com/google/gcloud/datastore/QueryResult.java +++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java @@ -3,31 +3,46 @@ import java.util.Iterator; /** - * A result of Google Cloud Datastore query. + * The result of a Google Cloud Datastore query submission. + * Typically the result's type would be casted to its expected type (The {@link #getResultType()} + * method could be used when the type is not known). + * + * @param V the type of values the result holds. */ public interface QueryResult extends Iterator { /** - * The type of the result. - * Full: A complete {@link Entity}. + * The result's type. + * FULL: A complete {@link Entity}. * PROJECTION: A partial entity, represented by {@link PartialEntity}. * KEY_ONLY: An entity's {@link Key}. */ - enum ResultType { - FULL, - PROJECTION, - KEY_ONLY; + enum Type { + + FULL(Entity.class), + PROJECTION(PartialEntity.class), + KEY_ONLY(Key.class); + + private final Class resultClass; + + Type(Class resultClass) { + this.resultClass = resultClass; + } + + Class getResultClass() { + return resultClass; + } } /** - * This method should be used to cast the result to its appropriate type. + * This method can be used to verify the result type and to cast its value type accordingly. * 
 {@code
-   * ResultType.FULL -> (QueryResult)
-   * ResultType.PROJECTION -> (QueryResult)
-   * ResultType.KEY_ONLY -> (QueryResult)
+   * Type.FULL -> (QueryResult)
+   * Type.PROJECTION -> (QueryResult)
+   * Type.KEY_ONLY -> (QueryResult)
    * }
*/ - ResultType getResultType(); + Type getType(); /** * Return the Cursor for the next result. Not currently implemented (depends on v1beta3). diff --git a/src/main/java/com/google/gcloud/datastore/RawValue.java b/src/main/java/com/google/gcloud/datastore/RawValue.java new file mode 100644 index 000000000000..120ad7085e4e --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/RawValue.java @@ -0,0 +1,59 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; + +public final class RawValue extends Value { + + private static final long serialVersionUID = -3359604598651897941L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public Builder newBuilder(DatastoreV1.Value value) { + return new Builder(value); + } + + @Override + public int getProtoFieldId() { + return 0; + } + + @Override + protected DatastoreV1.Value getValue(DatastoreV1.Value from) { + return from; + } + + @Override + protected void setValue(RawValue from, DatastoreV1.Value.Builder to) { + to.mergeFrom(from.get()); + } + }; + + static final class Builder extends Value.BaseBuilder { + + Builder(DatastoreV1.Value valuePb) { + super(Type.RAW_VALUE); + if (valuePb.hasIndexed()) { + indexed(valuePb.getIndexed()); + } + if (valuePb.hasMeaning()) { + meaning(valuePb.getMeaning()); + } + set(valuePb); + } + + @Override + public RawValue build() { + return new RawValue(this); + } + } + + RawValue(Builder builder) { + super(builder); + } + + RawValue(DatastoreV1.Value valuePb) { + this(new Builder(valuePb)); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Serializable.java b/src/main/java/com/google/gcloud/datastore/Serializable.java index c9f6b7984512..e9964b6d0e4a 100644 --- a/src/main/java/com/google/gcloud/datastore/Serializable.java +++ b/src/main/java/com/google/gcloud/datastore/Serializable.java @@ -15,6 +15,23 @@ abstract class Serializable implements java.io.Seria private transient byte[] bytesPb; // only for deserialization + + @Override + public int hashCode() { + return toPb().hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!getClass().isInstance(obj)) { + return false; + } + return toPb().equals(((Serializable) obj).toPb()); + } + @Override public String toString() { return toPb().toString(); diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java index 70d77e250782..7f8ee6e979d8 100644 --- a/src/main/java/com/google/gcloud/datastore/StringValue.java +++ b/src/main/java/com/google/gcloud/datastore/StringValue.java @@ -42,15 +42,15 @@ public Builder(String value) { @Override public StringValue build() { - if (getIndexed()) { + if (Boolean.TRUE.equals(getIndexed())) { checkArgument(get().length() <= 500, "Indexed string is limited to 500 characters"); } return new StringValue(this); } } - public StringValue(String content) { - this(new Builder(content)); + public StringValue(String value) { + this(new Builder(value)); } StringValue(Builder builder) { diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java index 4df12134c3ed..fd916960d78a 100644 --- a/src/main/java/com/google/gcloud/datastore/Transaction.java +++ b/src/main/java/com/google/gcloud/datastore/Transaction.java @@ -2,10 +2,20 @@ /** * A Google cloud datastore transaction. + * + * @see Google Cloud Datastore transactions */ public interface Transaction extends DatastoreReader, DatastoreWriter { + /** + * Commit the transaction. + * + * @throws DatastoreServiceException if could not commit the transaction + */ void commit(); + /** + * Rollback the transaction. + */ void rollback(); } diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java index edc197ddd696..3d24423b65d8 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -4,52 +4,111 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.MoreObjects; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.InvalidProtocolBufferException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; -// TODO: add javadoc, and mention that null should only be represented by NullValue. +/** + * Base class for all Google Cloud Datastore value types. + * All values must be associated with a non-null content (except {@link NullValue}). + * All values are immutable (including their content). To edit (a copy) use {@link #builder()}. + * Unsupported value (deprecated or unrecognized) would be represented by {@link RawValue}. + * + * @param the type of the content for this value + * @param

the type of this value + * @param the type of this value's builder + */ public abstract class Value, B extends Value.Builder> extends Serializable { private static final long serialVersionUID = -1899638277588872742L; + private static final Map DESCRIPTOR_TO_TYPE_MAP = new HashMap<>(); private final transient Type type; - private final transient boolean indexed; + private final transient Boolean indexed; private final transient Integer meaning; private final transient V value; + /** + * The type of a property. + * + * @see Google Cloud Datastore types + */ public enum Type { + /** + * Represents a {@code null} value. + */ NULL(NullValue.MARSHALLER, NullValue.MARSHALLER), + + /** + * Represents a {@code string} value. + */ STRING(StringValue.MARSHALLER, StringValue.MARSHALLER), + + /** + * Represents an {@link PartialEntity} value. + */ PARTIAL_ENTITY(PartialEntityValue.MARSHALLER, PartialEntityValue.MARSHALLER), + + /** + * Represents a {@code list} of {@link Value}s. + */ LIST(ListValue.MARSHALLER, ListValue.MARSHALLER), - KEY(KeyValue.MARSHALLER, KeyValue.MARSHALLER); - - /* - TODO: Also implement - LONG(LongValue.class, INTEGER_VALUE_FIELD_NUMBER), - DOUBLE(DoubleValue.class, DOUBLE_VALUE_FIELD_NUMBER), - // TODO: make sure that getContent returns an immutable value or at least a copy - TIMESTAMP(TimestampValue.class, TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER), - BOOLEAN(BooleanValue.class, BOOLEAN_VALUE_FIELD_NUMBER), - BLOB(BlobValue.class, BLOB_VALUE_FIELD_NUMBER), - BLOB_KEY(BlobKeyValue.class, BLOB_KEY_VALUE_FIELD_NUMBER), - // GEO_POINT(GeoPointValue.class, 8) // Does not seem to be public yet... + + /** + * Represents a {@code key} as a value. */ + KEY(KeyValue.MARSHALLER, KeyValue.MARSHALLER), + + /** + * Represents a {@code long} value. + */ + LONG(LongValue.MARSHALLER, LongValue.MARSHALLER), + + /** + * Represents a {@code double} value. + */ + DOUBLE(DoubleValue.MARSHALLER, DoubleValue.MARSHALLER), + + /** + * Represents a {@code boolean} value. + */ + BOOLEAN(BooleanValue.MARSHALLER, BooleanValue.MARSHALLER), + + /** + * Represents a {@link DateAndTime} value. + */ + DATE_AND_TIME(DateAndTimeValue.MARSHALLER, DateAndTimeValue.MARSHALLER), + + /** + * Represents a {@link Blob} value. + */ + BLOB(BlobValue.MARSHALLER, BlobValue.MARSHALLER), + + /** + * Represents a raw/unparsed value. + */ + RAW_VALUE(RawValue.MARSHALLER, RawValue.MARSHALLER); + @SuppressWarnings("rawtypes") private final BuilderFactory builderFactory; @SuppressWarnings("rawtypes") private final Marshaller marshaller; - private FieldDescriptor field; , B extends Builder> Type(Marshaller marshaller, BuilderFactory builderFactory) { this.marshaller = marshaller; this.builderFactory = builderFactory; - field = DatastoreV1.Value.getDescriptor().findFieldByNumber(marshaller.getProtoFieldId()); + int fieldId = marshaller.getProtoFieldId(); + if (fieldId > 0) { + DESCRIPTOR_TO_TYPE_MAP.put(fieldId, this); + } } , B extends Builder> Marshaller @@ -61,10 +120,6 @@ public enum Type { getBuilderFactory() { return builderFactory; } - - FieldDescriptor getDescriptor() { - return field; - } } interface BuilderFactory, B extends Builder> { @@ -78,7 +133,7 @@ interface Builder, B extends Builder> { B mergeFrom(P other); - boolean getIndexed(); + Boolean getIndexed(); B indexed(boolean indexed); @@ -108,7 +163,9 @@ abstract static class BaseMarshaller, B extends Buil @Override public final B fromProto(DatastoreV1.Value proto) { B builder = newBuilder(getValue(proto)); - builder.indexed(proto.getIndexed()); + if (proto.hasIndexed()) { + builder.indexed(proto.getIndexed()); + } if (proto.hasMeaning()) { builder.meaning(proto.getMeaning()); } @@ -118,9 +175,11 @@ public final B fromProto(DatastoreV1.Value proto) { @Override public final DatastoreV1.Value toProto(P value) { DatastoreV1.Value.Builder builder = DatastoreV1.Value.newBuilder(); - builder.setIndexed(value.getIndexed()); - if (value.getMeaning() != null) { - builder.setMeaning(value.getMeaning()); + if (value.hasIndexed()) { + builder.setIndexed(value.indexed()); + } + if (value.hasMeaning()) { + builder.setMeaning(value.meaning()); } setValue(value, builder); return builder.build(); @@ -135,7 +194,7 @@ abstract static class BaseBuilder, B extends BaseBui implements Builder { private final Type type; - private boolean indexed = true; + private Boolean indexed; private Integer meaning; private V value; @@ -150,14 +209,14 @@ public Type getType() { @Override public B mergeFrom(P other) { - indexed = other.getIndexed(); - meaning = other.getMeaning(); + indexed = other.indexed(); + meaning = other.meaning(); set(other.get()); return self(); } @Override - public boolean getIndexed() { + public Boolean getIndexed() { return indexed; } @@ -203,27 +262,34 @@ private B self() { indexed = builder.getIndexed(); meaning = builder.getMeaning(); // some validations: - if (meaning != null) { + if (meaning != null && indexed != null) { // TODO: consider supplying Ranges for allowed meaning and validating it here // more specific validation could be done on the specific types themselves // upon construction [e.g. integer with a meaning 13 should be in the range [0,100]] - if (indexed) { - checkArgument(meaning != 15 && meaning != 22, - "Indexed values should not have meaning with 15 or 22"); - } + checkArgument(!indexed || meaning != 15 && meaning != 22, + "Indexed values should not have meaning with 15 or 22"); } value = builder.get(); } - public final Type getType() { + public final Type type() { return type; } - public final boolean getIndexed() { - return indexed; + public final boolean hasIndexed() { + return indexed != null; + } + + public final boolean indexed() { + // default indexed value is true + return MoreObjects.firstNonNull(indexed, Boolean.TRUE); } - public final Integer getMeaning() { + public final boolean hasMeaning() { + return meaning != null; + } + + public final Integer meaning() { return meaning; } @@ -232,8 +298,8 @@ public final V get() { } @SuppressWarnings("unchecked") - public final B toBuilder() { - BuilderFactory builderFactory = getType().getBuilderFactory(); + public final B builder() { + BuilderFactory builderFactory = type().getBuilderFactory(); B builder = builderFactory.newBuilder(get()); return builder.mergeFrom((P) this); } @@ -245,48 +311,40 @@ public int hashCode() { @SuppressWarnings("unchecked") @Override - public boolean equals(Object other) { - if (!getClass().isInstance(other)) { + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!getClass().isInstance(obj)) { return false; } - - Value otherValue = (Value) other; - return Objects.equals(type, otherValue.getType()) - && Objects.equals(indexed, otherValue.getIndexed()) - && Objects.equals(meaning, otherValue.getMeaning()) - && Objects.equals(value, otherValue.get()); + Value other = (Value) obj; + return Objects.equals(type, other.type) + && Objects.equals(indexed, other.indexed) + && Objects.equals(meaning, other.meaning) + && Objects.equals(value, other.value); } @Override @SuppressWarnings("unchecked") protected DatastoreV1.Value toPb() { - Marshaller marshaller = getType().getMarshaller(); + Marshaller marshaller = type().getMarshaller(); return marshaller.toProto((P) this); } - @SuppressWarnings({ "rawtypes", "unchecked" }) - static , B extends Value.Builder> Value - fromPb(DatastoreV1.Value proto) { - Marshaller marshaller = NullValue.MARSHALLER; - for (Type type : Type.values()) { - FieldDescriptor descriptor = type.getDescriptor(); - if (descriptor != null) { - if (descriptor.isRepeated()) { - if (proto.getRepeatedFieldCount(descriptor) > 0) { - marshaller = type.getMarshaller(); - break; - } - } else if (proto.hasField(descriptor)) { - marshaller = type.getMarshaller(); - break; + static Value fromPb(DatastoreV1.Value proto) { + for (Entry entry : proto.getAllFields().entrySet()) { + FieldDescriptor descriptor = entry.getKey(); + if (descriptor.getName().endsWith("_value")) { + Type type = DESCRIPTOR_TO_TYPE_MAP.get(descriptor.getNumber()); + if (type == null) { + // unsupported type + return RawValue.MARSHALLER.fromProto(proto).build(); } + return type.getMarshaller().fromProto(proto).build(); } } - // change strategy to return RawValue (package scope constructor) - // when no match instead of null. This could only be done - // when using the V3 API which added a NullValue type to distinct the cases - // and the use of oneof which generates an enum of all possible values. - return marshaller.fromProto(proto).build(); + return NullValue.MARSHALLER.fromProto(proto).build(); } @Override diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 7f4f497dd970..ab8e839c15b2 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -1,6 +1,6 @@ package com.google.gcloud.datastore; -import static org.junit.Assert.*; +import static org.junit.Assert.fail; import org.junit.Test; @@ -65,5 +65,4 @@ public void testDelete() { public void testNewKeyBuilder() { fail("Not yet implemented"); } - } diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 0184f26717b9..2e44315c715d 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -1,8 +1,11 @@ package com.google.gcloud.datastore; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; -import com.google.common.collect.ImmutableMap; +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Multimap; import com.google.gcloud.datastore.Value.Type; import org.junit.Test; @@ -11,7 +14,6 @@ import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; -import java.util.Map; public class SerializationTest { @@ -19,53 +21,73 @@ public class SerializationTest { new PartialKey.Builder("ds", "k").addToPath("p", 1).build(); private static final Key KEY1 = new Key.Builder("ds", "k", "n").build(); private static final Key KEY2 = new Key.Builder(INCOMPLETE_KEY, 2).build(); - private static final KeyValue KEY_PROPERTY = new KeyValue(KEY1); - private static final NullValue NULL_PROPERTY = + private static final KeyValue KEY_VALUE = new KeyValue(KEY1); + private static final NullValue NULL_VALUE = new NullValue.Builder().indexed(true).build(); - private static final StringValue STRING_PROPERTY = new StringValue("hello"); + private static final StringValue STRING_VALUE = new StringValue("hello"); + private static final LongValue LONG_VALUE = new LongValue(123); + private static final DoubleValue DOUBLE_VALUE = new DoubleValue(12.34); + private static final BooleanValue BOOLEAN_VALUE = new BooleanValue(true); + private static final DateAndTimeValue DATE_AND_TIME_VALUE = + new DateAndTimeValue(DateAndTime.now()); + private static final BlobValue BLOB_VALUE = + new BlobValue(Blob.copyFrom(new byte[] {10, 0, -2})); + private static final RawValue RAW_VALUE = new RawValue( + DatastoreV1.Value.newBuilder().setBlobKeyValue("blob-key").setMeaning(18).build()); private static final Entity ENTITY1 = new Entity.Builder(KEY1).build(); private static final Entity ENTITY2 = new Entity.Builder(KEY2).setProperty("null", new NullValue()).build(); + private static final Entity ENTITY3 = + new Entity.Builder(KEY2) + .setProperty("p1", new StringValue.Builder("hi1").meaning(10).build()) + .setProperty("p2", new StringValue.Builder("hi2").meaning(11).indexed(false).build()) + .setProperty("p3", new LongValue.Builder(100).indexed(false).meaning(100).build()) + .build(); private static final PartialEntity EMBEDDED_ENTITY1 = ENTITY1; private static final PartialEntity EMBEDDED_ENTITY2 = ENTITY2; private static final PartialEntity EMBEDDED_ENTITY3 = - new PartialEntity.Builder() - .setKey(INCOMPLETE_KEY) - .setProperty("p1", STRING_PROPERTY) - .setProperty("p2", STRING_PROPERTY) + new PartialEntity.Builder(INCOMPLETE_KEY) + .setProperty("p1", STRING_VALUE) + .setProperty("p2", new LongValue.Builder(100).indexed(false).meaning(100).build()) .build(); - private static final PartialEntityValue EMBEDDED_ENTITY_PROPERTY1 = + private static final PartialEntityValue EMBEDDED_ENTITY_VALUE1 = new PartialEntityValue(EMBEDDED_ENTITY1); - private static final PartialEntityValue EMBEDDED_ENTITY_PROPERTY2 = + private static final PartialEntityValue EMBEDDED_ENTITY_VALUE2 = new PartialEntityValue(EMBEDDED_ENTITY2); - private static final PartialEntityValue EMBEDDED_ENTITY_PROPERTY3 = + private static final PartialEntityValue EMBEDDED_ENTITY_VALUE3 = new PartialEntityValue(EMBEDDED_ENTITY3); - private static final ListValue PROPERTY_LIST_PROPERTY = + private static final ListValue LIST_VALUE = new ListValue.Builder() - .addValue(NULL_PROPERTY) - .addValue(STRING_PROPERTY) + .addValue(NULL_VALUE) + .addValue(STRING_VALUE) .addValue(new NullValue()) .build(); - private Map typeToValues = ImmutableMap.of( - Type.NULL, array(NULL_PROPERTY), - Type.KEY, array(KEY_PROPERTY), - Type.STRING, array(STRING_PROPERTY), - Type.PARTIAL_ENTITY, array(EMBEDDED_ENTITY_PROPERTY1, EMBEDDED_ENTITY_PROPERTY2, - EMBEDDED_ENTITY_PROPERTY3), - Type.LIST, array(PROPERTY_LIST_PROPERTY)); - - private static T[] array(T... values) { - return values; - } + @SuppressWarnings("rawtypes") + private Multimap typeToValues = ImmutableMultimap.builder() + .put(Type.NULL, NULL_VALUE) + .put(Type.KEY, KEY_VALUE) + .put(Type.STRING, STRING_VALUE) + .putAll(Type.PARTIAL_ENTITY, EMBEDDED_ENTITY_VALUE1, EMBEDDED_ENTITY_VALUE2, + EMBEDDED_ENTITY_VALUE3) + .put(Type.LIST, LIST_VALUE) + .put(Type.LONG, LONG_VALUE) + .put(Type.DOUBLE, DOUBLE_VALUE) + .put(Type.BOOLEAN, BOOLEAN_VALUE) + .put(Type.DATE_AND_TIME, DATE_AND_TIME_VALUE) + .put(Type.BLOB, BLOB_VALUE) + .put(Type.RAW_VALUE, RAW_VALUE) + .build(); @Test public void testValues() throws Exception { for (Type type : Type.values()) { - Value[] values = typeToValues.get(type); - for (Value value : values) { + for (Value value : typeToValues.get(type)) { Value copy = serialiazeAndDeserialize(value); + assertEquals(value, value); assertEquals(value, copy); + assertNotSame(value, copy); + assertEquals(copy, copy); assertEquals(value.get(), copy.get()); } } @@ -73,25 +95,27 @@ public void testValues() throws Exception { @Test public void testTypes() throws Exception { - Object[] types = { KEY1, KEY2, INCOMPLETE_KEY, ENTITY1, ENTITY2, EMBEDDED_ENTITY1, - EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; + Object[] types = { KEY1, KEY2, INCOMPLETE_KEY, ENTITY1, ENTITY2, ENTITY3, + EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; for (Object obj : types) { Object copy = serialiazeAndDeserialize(obj); + assertEquals(obj, obj); assertEquals(obj, copy); + assertNotSame(obj, copy); + assertEquals(copy, copy); } } + @SuppressWarnings("unchecked") private T serialiazeAndDeserialize(T obj) throws Exception { - ByteArrayOutputStream bOut = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(bOut); - out.writeObject(obj); - out.close(); - byte[] bytes = bOut.toByteArray(); - ByteArrayInputStream bIn = new ByteArrayInputStream(bytes); - ObjectInputStream in = new ObjectInputStream(bIn); - @SuppressWarnings("unchecked") - T copy = (T) in.readObject(); - in.close(); - return copy; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try (ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream)) { + objectOutputStream.writeObject(obj); + } + byte[] bytes = byteArrayOutputStream.toByteArray(); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { + return (T) in.readObject(); + } } } From 9baa7214cad1d0f1d6833753944b35e98ba58d29 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 3 Dec 2014 20:34:01 -0800 Subject: [PATCH 026/732] work in progress --- .../datastore/DatastoreServiceFactory.java | 2 +- .../com/google/gcloud/datastore/Entity.java | 7 ++ .../java/com/google/gcloud/datastore/Key.java | 47 ++++++----- .../google/gcloud/datastore/KeyBuilder.java | 16 +++- .../gcloud/datastore/PartialEntity.java | 10 ++- .../gcloud/datastore/PartialEntityValue.java | 4 +- .../google/gcloud/datastore/PartialKey.java | 55 ++++++++---- .../datastore/DatastoreServiceTest.java | 83 +++++++++++++++++-- .../gcloud/datastore/SerializationTest.java | 14 ++-- 9 files changed, 181 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java index 903e33df5bf9..8d9f88a79ffe 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java @@ -3,7 +3,7 @@ public class DatastoreServiceFactory { - public DatastoreService getDatastoreService(DatastoreServiceOptions options) { + public static DatastoreService getDatastoreService(DatastoreServiceOptions options) { return new DatastoreServiceImpl(options); } } diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index 2bdd2dbfe41b..979174942e06 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -28,6 +28,13 @@ public Builder(Entity entity) { super(entity); } + /** + * Create a Builder for the given key and with the properties from the given entity. + */ + public Builder(Key key, PartialEntity entity) { + super(key, entity); + } + @Override public Builder clearProperties() { super.clearProperties(); diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index 33f013e8ad3f..96058259d30c 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -37,40 +37,37 @@ public Builder(String dataset, String kind, long id) { this.id = id; } - public Builder(PartialKey key, String name) { - super(key); + public Builder(Key parent, String kind, String name) { + super(parent, kind); this.name = name; } - public Builder(PartialKey key, long id) { - super(key); + public Builder(Key parent, String kind, long id) { + super(parent, kind); this.id = id; } - public Builder(Key key) { - super(key); - if (key.id() == null) { - name = key.name(); - } else { - id = key.id(); - } + @Override + public Builder addAncestor(String kind, long id) { + super.addAncestor(kind, id); + return this; } @Override - public Builder addToPath(String kind, long id) { - super.addToPath(kind, id); + public Builder addAncestor(String kind, String name) { + super.addAncestor(kind, name); return this; } @Override - public Builder addToPath(String kind, String name) { - super.addToPath(kind, name); + public Builder addAncestor(PathElement... ancestor) { + super.addAncestor(ancestor); return this; } @Override - public Builder addToPath(PathElement pathElement) { - super.addToPath(pathElement); + public Builder addAncestors(Iterable ancestors) { + super.addAncestors(ancestors); return this; } @@ -127,7 +124,13 @@ private Key(PartialKey key, long id) { @Override public Builder builder() { - return new Builder(this); + Builder builder = + hasId() ? new Builder(dataset(), kind(), id()) : new Builder(dataset(), kind(), name()); + return builder.namespace(namespace()).addAncestors(ancestors()); + } + + public boolean hasId() { + return id() != null; } /** @@ -137,6 +140,10 @@ public Long id() { return getLeaf().id(); } + public boolean hasName() { + return name() != null; + } + /** * Returns the key's name or {@code null} if it has an id instead. */ @@ -210,14 +217,14 @@ static Key fromPb(DatastoreV1.Key keyPb) { private static ImmutableList newPath(PartialKey key, String name) { ImmutableList.Builder path = ImmutableList.builder(); - path.addAll(key.ancestorPath()); + path.addAll(key.ancestors()); path.add(new PathElement(key.kind(), name)); return path.build(); } private static ImmutableList newPath(PartialKey key, long id) { ImmutableList.Builder path = ImmutableList.builder(); - path.addAll(key.ancestorPath()); + path.addAll(key.ancestors()); path.add(new PathElement(key.kind(), id)); return path.build(); } diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java index 07c6382f0dc5..39c8653a6d64 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -20,12 +20,12 @@ public KeyBuilder(DatastoreService service, String kind) { } public KeyBuilder addToPath(String kind, String name) { - delegate.addToPath(kind, name); + delegate.addAncestor(kind, name); return this; } public KeyBuilder addToPath(String kind, long id) { - delegate.addToPath(kind, id); + delegate.addAncestor(kind, id); return this; } @@ -43,11 +43,19 @@ public Key allocateIdAndBuild() { } public Key build(String name) { - return new Key.Builder(build(), name).build(); + PartialKey key = build(); + return new Key.Builder(key.dataset(), key.kind(), name) + .namespace(key.namespace()) + .addAncestors(key.ancestors()) + .build(); } public Key build(long id) { - return new Key.Builder(build(), id).build(); + PartialKey key = build(); + return new Key.Builder(key.dataset(), key.kind(), id) + .namespace(key.namespace()) + .addAncestors(key.ancestors()) + .build(); } public PartialKey build() { diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index c13ea5679b64..7d22be430372 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -41,6 +41,11 @@ public Builder(PartialEntity entity) { properties = new HashMap<>(entity.properties()); } + public Builder(PartialKey key, PartialEntity entity) { + this.key = key; + properties = new HashMap<>(entity.properties()); + } + public Builder clearProperties() { properties.clear(); return this; @@ -78,9 +83,8 @@ public boolean hasProperty(String name) { } @SuppressWarnings("unchecked") - public , B extends Value.Builder> Value property( - String name) { - return (Value) properties.get(name); + public > V property(String name) { + return (V) properties.get(name); } public Set propertyNames() { diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java b/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java index cc4dd834a343..db076a3305a0 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java @@ -5,7 +5,7 @@ import com.google.api.services.datastore.DatastoreV1; public class PartialEntityValue extends -Value { + Value { private static final long serialVersionUID = -5461475706792576395L; @@ -34,7 +34,7 @@ protected void setValue(PartialEntityValue from, DatastoreV1.Value.Builder to) { }; public static final class Builder extends - Value.BaseBuilder { + Value.BaseBuilder { public Builder(PartialEntity entity) { super(Type.PARTIAL_ENTITY); diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 4bcca81ddb64..7827b66dcd33 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -135,27 +135,38 @@ public Builder(String dataset, String kind) { this.kind = validateKind(kind); } - public Builder(PartialKey key) { - dataset = key.dataset; - namespace = key.namespace; - kind = key.kind(); - path.addAll(key.ancestorPath()); + public Builder(Key parent, String kind) { + dataset = parent.dataset(); + namespace = parent.namespace(); + path.addAll(parent.ancestors()); + path.add(parent.getLeaf()); + this.kind = kind; } - public Builder addToPath(String kind, long id) { + public Builder addAncestor(String kind, long id) { checkArgument(id != 0, "id must not be equal to zero"); - return addToPath(new PathElement(kind, id)); + return addAncestor(new PathElement(kind, id)); } - public Builder addToPath(String kind, String name) { + public Builder addAncestor(String kind, String name) { checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); checkArgument(name.length() <= 500, "name must not exceed 500 characters"); - return addToPath(new PathElement(kind, name)); + return addAncestor(new PathElement(kind, name)); + } + + public Builder addAncestor(PathElement... ancestor) { + Preconditions.checkState(path.size() + ancestor.length <= MAX_PATH, + "path can have at most 100 elements"); + for (PathElement pathElement : ancestor) { + path.add(pathElement); + } + return this; } - public Builder addToPath(PathElement pathElement) { - Preconditions.checkState(path.size() < MAX_PATH, "path can have at most 100 elements"); - path.add(pathElement); + public Builder addAncestors(Iterable ancestors) { + for (PathElement pathElement : ancestors) { + addAncestor(pathElement); + } return this; } @@ -215,9 +226,9 @@ public String namespace() { } /** - * Returns the key's parent's path. + * Returns an immutable list with the key's ancestors. */ - public List ancestorPath() { + public List ancestors() { return path.subList(0, path.size() - 1); } @@ -229,7 +240,21 @@ public String kind() { } public Builder builder() { - return new Builder(this); + return new Builder(dataset(), kind()).namespace(namespace()).addAncestors(ancestors()); + } + + public Key toKey(String name) { + return new Key.Builder(dataset(), kind(), name) + .namespace(namespace()) + .addAncestors(ancestors()) + .build(); + } + + public Key toKey(long id) { + return new Key.Builder(dataset(), kind(), id) + .namespace(namespace()) + .addAncestors(ancestors()) + .build(); } @Override diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index ab8e839c15b2..e7944009eebb 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -1,14 +1,52 @@ package com.google.gcloud.datastore; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import org.junit.Before; import org.junit.Test; +import java.util.Iterator; + public class DatastoreServiceTest { + private static final String DATASET = "dataset1"; + private static final String KIND1 = "kind1"; + private static final String KIND2 = "kind2"; + private static final NullValue NULL_VALUE = new NullValue(); + private static final StringValue STR_VALUE = new StringValue("str"); + private static final BooleanValue BOOL_VALUE = + new BooleanValue.Builder(false).indexed(false).build(); + private static final PartialKey PARTIAL_KEY1 = new PartialKey.Builder(DATASET, KIND1).build(); + private static final PartialKey PARTIAL_KEY2 = new PartialKey.Builder(DATASET, KIND2).build(); + private static final Key KEY1 = PARTIAL_KEY1.toKey("name"); + private static final Key KEY2 = new Key.Builder(KEY1, KIND2, 1).build(); + private static final Key KEY3 = KEY2.builder().name("bla").build(); + private static final Entity ENTITY1 = new Entity.Builder(KEY1) + .setProperty("str", STR_VALUE) + .setProperty("bool", BOOL_VALUE) + .build(); + private static final Entity ENTITY2 = new Entity.Builder(KEY2, ENTITY1) + .removeProperty("str") + .setProperty("null", NULL_VALUE) + .build(); + + private DatastoreServiceOptions options; + private DatastoreService datastore; + + @Before + public void setUp() { + options = new DatastoreServiceOptions.Builder().dataset(DATASET).build(); + datastore = DatastoreServiceFactory.getDatastoreService(options); + } + @Test public void testGetOptions() { - fail("Not yet implemented"); + assertSame(options, datastore.getOptions()); } @Test @@ -32,13 +70,41 @@ public void testAllocateIds() { } @Test - public void testGetKey() { - fail("Not yet implemented"); + public void testAddAndGet() { + Entity entity = datastore.get(KEY1); + assertNull(entity); + + datastore.add(ENTITY1); + + entity = datastore.get(KEY1); + StringValue value1 = entity.property("str"); + BooleanValue value2 = entity.property("bool"); + assertEquals(value1, STR_VALUE); + assertEquals(value2, BOOL_VALUE); + assertEquals(2, entity.propertyNames().size()); + assertTrue(entity.propertyNames().contains("str")); + assertTrue(entity.propertyNames().contains("bool")); + assertFalse(entity.hasProperty("bla")); } @Test - public void testGetKeyArray() { - fail("Not yet implemented"); + public void testAddAndGetArray() { + Iterator result = datastore.get(KEY1, KEY2); + assertNull(result.next()); + assertNull(result.next()); + assertFalse(result.hasNext()); + + populateDatastore(); + + result = datastore.get(KEY1, KEY1.builder().name("bla").build(), KEY2); + assertEquals(ENTITY1, result.next()); + assertNull(result.next()); + assertEquals(ENTITY2, result.next()); + assertFalse(result.hasNext()); + } + + private void populateDatastore() { + datastore.add(ENTITY1, ENTITY2); } @Test @@ -63,6 +129,11 @@ public void testDelete() { @Test public void testNewKeyBuilder() { - fail("Not yet implemented"); + KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1); + assertEquals(PARTIAL_KEY1, keyBuilder.build()); + assertEquals(PARTIAL_KEY1.builder().kind(KIND2).build(), + datastore.newKeyBuilder(KIND2).build()); + assertEquals(KEY1, keyBuilder.build("name")); + assertEquals(KEY1.builder().id(2).build(), keyBuilder.build(2)); } } diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 2e44315c715d..d450d84f08b8 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -17,10 +17,12 @@ public class SerializationTest { - private static final PartialKey INCOMPLETE_KEY = - new PartialKey.Builder("ds", "k").addToPath("p", 1).build(); + private static final PartialKey INCOMPLETE_KEY1 = + new PartialKey.Builder("ds", "k").addAncestor("p", 1).build(); private static final Key KEY1 = new Key.Builder("ds", "k", "n").build(); - private static final Key KEY2 = new Key.Builder(INCOMPLETE_KEY, 2).build(); + private static final PartialKey INCOMPLETE_KEY2 = + new PartialKey.Builder(KEY1, "v").addAncestor("p", 1).build(); + private static final Key KEY2 = new Key.Builder(KEY1, "v", 2).build(); private static final KeyValue KEY_VALUE = new KeyValue(KEY1); private static final NullValue NULL_VALUE = new NullValue.Builder().indexed(true).build(); @@ -46,7 +48,7 @@ public class SerializationTest { private static final PartialEntity EMBEDDED_ENTITY1 = ENTITY1; private static final PartialEntity EMBEDDED_ENTITY2 = ENTITY2; private static final PartialEntity EMBEDDED_ENTITY3 = - new PartialEntity.Builder(INCOMPLETE_KEY) + new PartialEntity.Builder(INCOMPLETE_KEY1) .setProperty("p1", STRING_VALUE) .setProperty("p2", new LongValue.Builder(100).indexed(false).meaning(100).build()) .build(); @@ -95,8 +97,8 @@ public void testValues() throws Exception { @Test public void testTypes() throws Exception { - Object[] types = { KEY1, KEY2, INCOMPLETE_KEY, ENTITY1, ENTITY2, ENTITY3, - EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; + Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2, + ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; for (Object obj : types) { Object copy = serialiazeAndDeserialize(obj); assertEquals(obj, obj); From 8c1f432360305bc012cb4b067929479f2b9dc12f Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 3 Dec 2014 23:01:41 -0800 Subject: [PATCH 027/732] work in progress --- .../java/com/google/gcloud/AuthConfig.java | 8 +- .../com/google/gcloud/ServiceOptions.java | 27 +++-- .../datastore/DatastoreServiceException.java | 71 ++++++++++-- .../datastore/DatastoreServiceFactory.java | 34 +++++- .../datastore/DatastoreServiceImpl.java | 106 ++++++++++++++++-- .../datastore/DatastoreServiceOptions.java | 19 +++- .../google/gcloud/datastore/KeyBuilder.java | 4 +- .../datastore/DatastoreServiceTest.java | 21 +++- 8 files changed, 244 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java index d6321e5488d7..a6e47536e9e7 100644 --- a/src/main/java/com/google/gcloud/AuthConfig.java +++ b/src/main/java/com/google/gcloud/AuthConfig.java @@ -19,7 +19,7 @@ public abstract class AuthConfig { private static class AppEngineAuthConfig extends AuthConfig { @Override - protected HttpRequestInitializer getHttpRequestInitializer( + protected HttpRequestInitializer httpRequestInitializer( HttpTransport transport, Set scopes) { return new AppIdentityCredential(scopes); } @@ -36,7 +36,7 @@ public ServiceAccountAuthConfig(String account, PrivateKey privateKey) { } @Override - protected HttpRequestInitializer getHttpRequestInitializer( + protected HttpRequestInitializer httpRequestInitializer( HttpTransport transport, Set scopes) { return new GoogleCredential.Builder() .setTransport(transport) @@ -48,7 +48,7 @@ protected HttpRequestInitializer getHttpRequestInitializer( } } - protected abstract HttpRequestInitializer getHttpRequestInitializer( + protected abstract HttpRequestInitializer httpRequestInitializer( HttpTransport transport, Set scopes); @@ -60,7 +60,7 @@ public static AuthConfig createForComputeEngine() throws IOException, GeneralSec final ComputeCredential cred = getComputeCredential(); return new AuthConfig() { @Override - protected HttpRequestInitializer getHttpRequestInitializer(HttpTransport ts, Set sc) { + protected HttpRequestInitializer httpRequestInitializer(HttpTransport ts, Set sc) { return cred; } }; diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 137b9856bf0d..ff001b06fa6b 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -4,6 +4,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import com.google.api.client.extensions.appengine.http.UrlFetchTransport; +import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; @@ -63,7 +64,7 @@ protected static String getAppEngineAppId() { return System.getProperty("com.google.appengine.application.id"); } - protected abstract static class Builder { + protected abstract static class Builder> { private String host; private HttpTransport httpTransport; @@ -79,33 +80,37 @@ protected Builder(ServiceOptions options) { protected abstract ServiceOptions build(); - public Builder host(String host) { + public B host(String host) { this.host = host; - return this; + return (B) this; } - public Builder httpTransport(HttpTransport httpTransport) { + public B httpTransport(HttpTransport httpTransport) { this.httpTransport = httpTransport; - return this; + return (B) this; } - public Builder authConfig(AuthConfig authConfig) { + public B authConfig(AuthConfig authConfig) { this.authConfig = authConfig; - return this; + return (B) this; } } - protected abstract Set getScopes(); + protected abstract Set scopes(); - public String getHost() { + public String host() { return host; } - public HttpTransport getHttpTransport() { + public HttpTransport httpTransport() { return httpTransport; } - public AuthConfig getAuthConfig() { + public AuthConfig authConfig() { return authConfig; } + + protected HttpRequestInitializer httpRequestInitializer() { + return authConfig().httpRequestInitializer(httpTransport, scopes()); + } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java index 5b27cd519654..912efe4a4692 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java @@ -1,20 +1,77 @@ package com.google.gcloud.datastore; +import static com.google.common.base.MoreObjects.firstNonNull; + +import com.google.api.services.datastore.client.DatastoreException; + +import java.util.HashMap; +import java.util.Map; + public class DatastoreServiceException extends RuntimeException { private static final long serialVersionUID = 8170357898917041899L; + private static final Map HTTP_TO_CODE = new HashMap<>(); + + private final Code code; + + /** + * An error code to represent the failure. + * + * @see Google Cloud Datastore error codes + */ + public enum Code { + + ABORTED(409, true, "Request aborted"), + DEADLINE_EXCEEDED(403, true, "Deadline exceeded"), + UNAVAILABLE(503, true, "Could not reach service"), + FAILED_PRECONDITION(412, false, "Invalid request"), + INVALID_ARGUMENT(400, false, "Request parameter has an invalid value"), + PERMISSION_DENIED(403, false, "Unauthorized request"), + RESOURCE_EXHAUSTED(402, false, "Quota exceeded"), + INTERNAL(500, false, "Server returned an error"), + UNKNOWN(0, false, "Unknown failure"); - private final boolean isTransient; + private final boolean isTransient; + private final String msg; - public DatastoreServiceException(boolean isTransient, String msg, Exception cause) { - super(msg, cause); - this.isTransient = isTransient; + Code(int httpStatus, boolean isTransient, String msg) { + this.isTransient = isTransient; + this.msg = msg; + HTTP_TO_CODE.put(httpStatus, this); + } + + /** + * Returns {@code true} if this exception is transient and the same request could be retried. + * For any retry it is highly recommended to apply an exponential backoff. + */ + public boolean isTransient() { + return isTransient; + } + + DatastoreServiceException translate(DatastoreException exception) { + return new DatastoreServiceException(this, exception); + } + } + + public DatastoreServiceException(Code code, Exception cause) { + super(code.msg, cause); + this.code = code; + } + + /** + * Returns the code associated with this exception. + */ + public Code code() { + return code; } /** - * @return {@code true} if this exception is transient and could be safely retried. + * Translate DatastoreException to DatastoreServiceException based on their + * HTTP error codes. This method will always throw a new DatastoreServiceException. + * + * @throws DatastoreServiceException for every given DatastoreException */ - public boolean isTransient() { - return isTransient; + static DatastoreServiceException translateAndPropagate(DatastoreException exception) { + throw firstNonNull(HTTP_TO_CODE.get(exception.getCode()), Code.UNKNOWN).translate(exception); } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java index 8d9f88a79ffe..4e69efe743d1 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java @@ -1,9 +1,37 @@ package com.google.gcloud.datastore; +import com.google.api.services.datastore.client.Datastore; +import com.google.api.services.datastore.client.DatastoreFactory; +import com.google.api.services.datastore.client.DatastoreOptions; +import com.google.api.services.datastore.client.LocalDevelopmentDatastoreFactory; -public class DatastoreServiceFactory { - public static DatastoreService getDatastoreService(DatastoreServiceOptions options) { - return new DatastoreServiceImpl(options); +public interface DatastoreServiceFactory { + + enum Mode implements DatastoreServiceFactory { + + TESTING { + + @Override + public DatastoreService get(DatastoreServiceOptions options) { + DatastoreOptions dsOptions = new DatastoreOptions.Builder() + .dataset(options.dataset()) + .host(options.host()) + .build(); + Datastore datastore = LocalDevelopmentDatastoreFactory.get().create(dsOptions); + return new DatastoreServiceImpl(options, datastore); + } + }, + + PROD { + @Override + public DatastoreService get(DatastoreServiceOptions options) { + DatastoreOptions dsOptions = options.toDatastoreOptions(); + Datastore datastore = DatastoreFactory.get().create(dsOptions); + return new DatastoreServiceImpl(options, datastore); + } + }; } + + DatastoreService get(DatastoreServiceOptions options); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index fb73952613dc..499a1893b243 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -1,14 +1,23 @@ package com.google.gcloud.datastore; +import com.google.api.services.datastore.DatastoreV1; +import com.google.api.services.datastore.client.Datastore; +import com.google.api.services.datastore.client.DatastoreException; +import com.google.common.collect.AbstractIterator; + +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; final class DatastoreServiceImpl implements DatastoreService { private final DatastoreServiceOptions options; + private final Datastore datastore; - DatastoreServiceImpl(DatastoreServiceOptions options) { + DatastoreServiceImpl(DatastoreServiceOptions options, Datastore datastore) { this.options = options; + this.datastore = datastore; } @Override @@ -35,9 +44,27 @@ public Key allocateId(PartialKey key) { @Override public Iterator allocateIds(PartialKey... key) { - // TODO Auto-generated method stub - // Will need to populate "force" after b/18594027 is fixed. - return null; + DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder(); + for (PartialKey k : key) { + requestPb.addKey(k.toPb()); + } + // TODO(ozarov): will need to populate "force" after b/18594027 is fixed. + try { + DatastoreV1.AllocateIdsResponse responsePb = datastore.allocateIds(requestPb.build()); + final Iterator keys = responsePb.getKeyList().iterator(); + return new AbstractIterator() { + + @Override + protected Key computeNext() { + if (keys.hasNext()) { + return Key.fromPb(keys.next()); + } + return endOfData(); + } + }; + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndPropagate(e); + } } @Override @@ -46,29 +73,82 @@ public Entity get(Key key) { } @Override - public Iterator get(Key... key) { - // TODO Auto-generated method stub - return null; + public Iterator get(final Key... key) { + DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder(); + for (Key k : key) { + requestPb.addKey(k.toPb()); + } + try { + DatastoreV1.LookupResponse responsePb = datastore.lookup(requestPb.build()); + final Map result = new HashMap<>(); + for (DatastoreV1.EntityResult entityResultPb : responsePb.getFoundList()) { + Entity entity = Entity.fromPb(entityResultPb.getEntity()); + result.put(entity.key(), entity); + } + return new AbstractIterator() { + int index; + + @Override + protected Entity computeNext() { + if (index < key.length) { + return result.get(key[index++]); + } + return endOfData(); + } + }; + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndPropagate(e); + } } @Override public void add(Entity... entity) { - // TODO Auto-generated method stub + DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + for (Entity e : entity) { + mutationPb.addInsert(e.toPb()); + } + comitMutation(mutationPb); + } + + private void comitMutation(DatastoreV1.Mutation.Builder mutationPb) { + if (options.force()) { + mutationPb.setForce(true); + } + DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); + requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL); + requestPb.setMutation(mutationPb.build()); + try { + datastore.commit(requestPb.build()); + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndPropagate(e); + } } @Override public void update(Entity... entity) { - // TODO Auto-generated method stub + DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + for (Entity e : entity) { + mutationPb.addUpdate(e.toPb()); + } + comitMutation(mutationPb); } @Override public void put(Entity... entity) { - // TODO Auto-generated method stub + DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + for (Entity e : entity) { + mutationPb.addUpsert(e.toPb()); + } + comitMutation(mutationPb); } @Override public void delete(Key... key) { - // TODO Auto-generated method stub + DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + for (Key k : key) { + mutationPb.addDelete(k.toPb()); + } + comitMutation(mutationPb); } @Override @@ -81,4 +161,8 @@ public QueryResult runQuery(Query query) { // TODO Auto-generated method stub return null; } + + Datastore datastore() { + return datastore; + } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 06377531b25a..74d218663c76 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -3,6 +3,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; +import com.google.api.services.datastore.client.DatastoreOptions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import com.google.gcloud.ServiceOptions; @@ -29,7 +30,7 @@ public class DatastoreServiceOptions extends ServiceOptions { force = builder.force; } - public static class Builder extends ServiceOptions.Builder { + public static class Builder extends ServiceOptions.Builder { private String dataset; private boolean force = false; @@ -69,11 +70,11 @@ static String validateDataset(String dataset) { return dataset; } - public String getDataset() { + public String dataset() { return dataset; } - public String getDefaultNamespace() { + public String defaultNamespace() { // TODO(ozarov): An alternative to reflection would be to depend on AE api jar: // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0 try { @@ -85,12 +86,20 @@ public String getDefaultNamespace() { } } - public boolean getForce() { + public boolean force() { return force; } @Override - protected Set getScopes() { + protected Set scopes() { return SCOPES; } + + DatastoreOptions toDatastoreOptions() { + return new DatastoreOptions.Builder() + .dataset(dataset()) + .host(host()) + .initializer(httpRequestInitializer()) + .build(); + } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java index 39c8653a6d64..11b6af1f909c 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -15,8 +15,8 @@ public final class KeyBuilder { */ public KeyBuilder(DatastoreService service, String kind) { this.service = checkNotNull(service); - delegate = new PartialKey.Builder(service.getOptions().getDataset(), kind); - delegate.namespace(service.getOptions().getDefaultNamespace()); + delegate = new PartialKey.Builder(service.getOptions().dataset(), kind); + delegate.namespace(service.getOptions().defaultNamespace()); } public KeyBuilder addToPath(String kind, String name) { diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index e7944009eebb..fb00bb4dfa44 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -7,6 +7,10 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.api.services.datastore.client.LocalDevelopmentDatastore; +import com.google.api.services.datastore.client.LocalDevelopmentDatastoreException; + +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -37,11 +41,22 @@ public class DatastoreServiceTest { private DatastoreServiceOptions options; private DatastoreService datastore; + private LocalDevelopmentDatastore localDatastore; @Before - public void setUp() { - options = new DatastoreServiceOptions.Builder().dataset(DATASET).build(); - datastore = DatastoreServiceFactory.getDatastoreService(options); + public void setUp() throws LocalDevelopmentDatastoreException { + options = new DatastoreServiceOptions.Builder() + .dataset(DATASET) + .host("http://localhost:8080/") + .build(); + datastore = DatastoreServiceFactory.Mode.TESTING.get(options); + localDatastore = (LocalDevelopmentDatastore) ((DatastoreServiceImpl) datastore).datastore(); + localDatastore.start("/usr/local/gcd-sdk", DATASET); + } + + @After + public void tearDown() throws LocalDevelopmentDatastoreException { + localDatastore.stop(); } @Test From 364f00abba08f6b13acd116f95d8bc2bac38b017 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 4 Dec 2014 16:26:04 -0800 Subject: [PATCH 028/732] Add more tests and package-info --- .../java/com/google/gcloud/AuthConfig.java | 20 ++- .../com/google/gcloud/ServiceOptions.java | 19 ++- .../gcloud/datastore/DatastoreService.java | 5 +- .../datastore/DatastoreServiceFactory.java | 25 +-- .../datastore/DatastoreServiceImpl.java | 7 + .../datastore/DatastoreServiceOptions.java | 51 ++++-- .../java/com/google/gcloud/datastore/Key.java | 20 +-- .../google/gcloud/datastore/KeyBuilder.java | 57 ++++--- .../google/gcloud/datastore/NullValue.java | 2 +- .../google/gcloud/datastore/PartialKey.java | 103 ++++++------ .../google/gcloud/datastore/package-info.java | 31 ++++ .../datastore/DatastoreServiceTest.java | 154 +++++++++++++----- 12 files changed, 326 insertions(+), 168 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/package-info.java diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java index a6e47536e9e7..117db5434a00 100644 --- a/src/main/java/com/google/gcloud/AuthConfig.java +++ b/src/main/java/com/google/gcloud/AuthConfig.java @@ -8,6 +8,7 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.client.repackaged.com.google.common.base.Preconditions; import java.io.IOException; import java.security.GeneralSecurityException; @@ -33,18 +34,23 @@ private static class ServiceAccountAuthConfig extends AuthConfig { public ServiceAccountAuthConfig(String account, PrivateKey privateKey) { this.account = account; this.privateKey = privateKey; + if (privateKey != null) { + Preconditions.checkArgument(account != null); + } } @Override protected HttpRequestInitializer httpRequestInitializer( HttpTransport transport, Set scopes) { - return new GoogleCredential.Builder() - .setTransport(transport) - .setJsonFactory(new JacksonFactory()) - .setServiceAccountId(account) - .setServiceAccountScopes(scopes) - .setServiceAccountPrivateKey(privateKey) - .build(); + GoogleCredential.Builder builder = new GoogleCredential.Builder() + .setTransport(transport) + .setJsonFactory(new JacksonFactory()) + .setServiceAccountId(account) + .setServiceAccountPrivateKey(privateKey); + if (privateKey != null) { + builder.setServiceAccountScopes(scopes); + } + return builder.build(); } } diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index ff001b06fa6b..908710ceb750 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -18,15 +18,15 @@ public abstract class ServiceOptions { private final HttpTransport httpTransport; private final AuthConfig authConfig; - protected ServiceOptions(Builder builder) { + protected ServiceOptions(Builder builder) { host = firstNonNull(builder.host, DEFAULT_HOST); - httpTransport = firstNonNull(builder.httpTransport, getDefaultHttpTransport()); - authConfig = firstNonNull(builder.authConfig, getDefaultAuthConfig()); + httpTransport = firstNonNull(builder.httpTransport, defaultHttpTransport()); + authConfig = firstNonNull(builder.authConfig, defaultAuthConfig()); } - private static HttpTransport getDefaultHttpTransport() { + private static HttpTransport defaultHttpTransport() { // Consider App Engine - if (getAppEngineAppId() != null) { + if (appEngineAppId() != null) { try { return new UrlFetchTransport(); } catch (Exception ignore) { @@ -42,9 +42,9 @@ private static HttpTransport getDefaultHttpTransport() { return new NetHttpTransport(); } - private static AuthConfig getDefaultAuthConfig() { + private static AuthConfig defaultAuthConfig() { // Consider App Engine - if (getAppEngineAppId() != null) { + if (appEngineAppId() != null) { try { return AuthConfig.createForAppEngine(); } catch (Exception ignore) { @@ -60,7 +60,7 @@ private static AuthConfig getDefaultAuthConfig() { return AuthConfig.createForAccount(null, null); } - protected static String getAppEngineAppId() { + protected static String appEngineAppId() { return System.getProperty("com.google.appengine.application.id"); } @@ -80,16 +80,19 @@ protected Builder(ServiceOptions options) { protected abstract ServiceOptions build(); + @SuppressWarnings("unchecked") public B host(String host) { this.host = host; return (B) this; } + @SuppressWarnings("unchecked") public B httpTransport(HttpTransport httpTransport) { this.httpTransport = httpTransport; return (B) this; } + @SuppressWarnings("unchecked") public B authConfig(AuthConfig authConfig) { this.authConfig = authConfig; return (B) this; diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 1b67fbe447eb..37fb52ad701f 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -32,7 +32,9 @@ public interface DatastoreService extends DatastoreReader, DatastoreWriter { BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption); /** - * Returns a key with a newly allocated id. + * Allocate a unique id for the given key. + * The returned key will have the same information (dataset, kind, namespace and ancestors) + * as the given key and will have a newly assigned id. * * @throws DatastoreServiceExcepiton upon failure */ @@ -41,6 +43,7 @@ public interface DatastoreService extends DatastoreReader, DatastoreWriter { /** * Returns a list of keys using the allocated ids ordered by the input. * + * @see #allocateId(PartialKey) * @throws DatastoreServiceExcepiton upon failure */ Iterator allocateIds(PartialKey... key); diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java index 4e69efe743d1..2ae4a3011836 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java @@ -3,27 +3,11 @@ import com.google.api.services.datastore.client.Datastore; import com.google.api.services.datastore.client.DatastoreFactory; import com.google.api.services.datastore.client.DatastoreOptions; -import com.google.api.services.datastore.client.LocalDevelopmentDatastoreFactory; -public interface DatastoreServiceFactory { +public abstract class DatastoreServiceFactory { - enum Mode implements DatastoreServiceFactory { - - TESTING { - - @Override - public DatastoreService get(DatastoreServiceOptions options) { - DatastoreOptions dsOptions = new DatastoreOptions.Builder() - .dataset(options.dataset()) - .host(options.host()) - .build(); - Datastore datastore = LocalDevelopmentDatastoreFactory.get().create(dsOptions); - return new DatastoreServiceImpl(options, datastore); - } - }, - - PROD { + private static final DatastoreServiceFactory INSTANCE = new DatastoreServiceFactory() { @Override public DatastoreService get(DatastoreServiceOptions options) { DatastoreOptions dsOptions = options.toDatastoreOptions(); @@ -31,7 +15,10 @@ public DatastoreService get(DatastoreServiceOptions options) { return new DatastoreServiceImpl(options, datastore); } }; + + public static DatastoreService getDefault(DatastoreServiceOptions options) { + return INSTANCE.get(options); } - DatastoreService get(DatastoreServiceOptions options); + public abstract DatastoreService get(DatastoreServiceOptions options); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 499a1893b243..4dcae9612568 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -46,6 +46,13 @@ public Key allocateId(PartialKey key) { public Iterator allocateIds(PartialKey... key) { DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder(); for (PartialKey k : key) { + if (k.getLeaf().nameOrId() != null) { + // if key is full remove the id or name part + k = new PartialKey.Builder(k.dataset(), k.kind()) + .namespace(k.namespace()) + .addAncestors(k.ancestors()) + .build(); + } requestPb.addKey(k.toPb()); } // TODO(ozarov): will need to populate "force" after b/18594027 is fixed. diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 74d218663c76..8b2f906283d9 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -17,22 +17,28 @@ public class DatastoreServiceOptions extends ServiceOptions { private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore"; private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email"; private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE); - private static final Pattern PATTERN = Pattern.compile( + private static final Pattern DATASET_PATTERN = Pattern.compile( "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}\\:)?([a-z\\d][a-z\\d\\-]{0,99})"); + private static final int MAX_NAMESPACE_LENGTH = 100; + private static final Pattern NAMESPACE_PATTERN = + Pattern.compile(String.format("[0-9A-Za-z\\._\\-]{0,%d}", MAX_NAMESPACE_LENGTH)); private final String dataset; + private final String namespace; private final boolean force; DatastoreServiceOptions(Builder builder) { super(builder); - dataset = firstNonNull(builder.dataset, getAppEngineAppId()); + dataset = firstNonNull(builder.dataset, appEngineAppId()); checkArgument(dataset != null, "missing dataset"); + namespace = builder.namespace != null ? builder.namespace : defaultNamespace(); force = builder.force; } public static class Builder extends ServiceOptions.Builder { private String dataset; + private String namespace; private boolean force = false; public Builder() {} @@ -53,36 +59,53 @@ public Builder dataset(String dataset) { return this; } + public Builder namespace(String namespace) { + this.namespace = validateNamespace(namespace); + return this; + } + public Builder force(boolean force) { this.force = force; return this; } } - static String validateDataset(String dataset) { - if (Strings.isNullOrEmpty(dataset)) { - throw new IllegalArgumentException("dataset can't be empty or null"); - } - if (!PATTERN.matcher(dataset).matches()) { - throw new IllegalArgumentException( - "dataset must match the following pattern: " + PATTERN.pattern()); - } + public String dataset() { return dataset; } - public String dataset() { + public String namespace() { + return namespace; + } + + static String validateDataset(String dataset) { + checkArgument(!Strings.isNullOrEmpty(dataset), "dataset can't be empty or null"); + checkArgument(DATASET_PATTERN.matcher(dataset).matches(), + "dataset must match the following pattern: " + DATASET_PATTERN.pattern()); return dataset; } - public String defaultNamespace() { + static String validateNamespace(String namespace) { + if (namespace != null) { + checkArgument(!namespace.isEmpty(), "namespace must not be an empty string"); + checkArgument(namespace.length() <= 100, + "namespace must not contain more than 100 characters"); + checkArgument(NAMESPACE_PATTERN.matcher(namespace).matches(), + "namespace must the following pattern: " + NAMESPACE_PATTERN.pattern()); + } + return namespace; + } + + private static String defaultNamespace() { // TODO(ozarov): An alternative to reflection would be to depend on AE api jar: // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0 try { Class clazz = Class.forName("com.google.appengine.api.NamespaceManager"); Method method = clazz.getMethod("get"); - return (String) method.invoke(null); + String namespace = (String) method.invoke(null); + return "".equals(namespace) ? null : namespace; } catch (Exception ex) { - return ""; + return null; } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index 96058259d30c..c490dded05f5 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -60,13 +60,13 @@ public Builder addAncestor(String kind, String name) { } @Override - public Builder addAncestor(PathElement... ancestor) { + public Builder addAncestor(Ancestor... ancestor) { super.addAncestor(ancestor); return this; } @Override - public Builder addAncestors(Iterable ancestors) { + public Builder addAncestors(Iterable ancestors) { super.addAncestors(ancestors); return this; } @@ -155,7 +155,7 @@ public String name() { * Returns the key's id (as {@link #Long}) or name (as {@link String}). */ public Object nameOrId() { - PathElement leaf = getLeaf(); + Ancestor leaf = getLeaf(); return leaf.hasId() ? leaf.id() : leaf.name(); } @@ -197,7 +197,7 @@ public static Key fromIncompleteKey(PartialKey key) { if (key instanceof Key) { return (Key) key; } - PathElement leaf = key.getLeaf(); + Ancestor leaf = key.getLeaf(); if (leaf.hasId()) { return new Key(key, leaf.id()); } else if (leaf.hasName()) { @@ -215,17 +215,17 @@ static Key fromPb(DatastoreV1.Key keyPb) { return fromIncompleteKey(PartialKey.fromPb(keyPb)); } - private static ImmutableList newPath(PartialKey key, String name) { - ImmutableList.Builder path = ImmutableList.builder(); + private static ImmutableList newPath(PartialKey key, String name) { + ImmutableList.Builder path = ImmutableList.builder(); path.addAll(key.ancestors()); - path.add(new PathElement(key.kind(), name)); + path.add(new Ancestor(key.kind(), name)); return path.build(); } - private static ImmutableList newPath(PartialKey key, long id) { - ImmutableList.Builder path = ImmutableList.builder(); + private static ImmutableList newPath(PartialKey key, long id) { + ImmutableList.Builder path = ImmutableList.builder(); path.addAll(key.ancestors()); - path.add(new PathElement(key.kind(), id)); + path.add(new Ancestor(key.kind(), id)); return path.build(); } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java index 11b6af1f909c..8b9954717858 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -2,44 +2,59 @@ import static com.google.common.base.Preconditions.checkNotNull; +import com.google.gcloud.datastore.PartialKey.Ancestor; + /** - * An helper for creating keys for a specific {@link DatastoreService}/dataset. + * An helper for creating keys for a specific {@link DatastoreService}, + * using its associated dataset and namespace. */ -public final class KeyBuilder { +public final class KeyBuilder extends PartialKey.Builder { - private final PartialKey.Builder delegate; private final DatastoreService service; /** * Constructing a KeyBuilder. */ public KeyBuilder(DatastoreService service, String kind) { - this.service = checkNotNull(service); - delegate = new PartialKey.Builder(service.getOptions().dataset(), kind); - delegate.namespace(service.getOptions().defaultNamespace()); + super(checkNotNull(service).getOptions().dataset(), kind); + this.service = service; + namespace(service.getOptions().namespace()); } - public KeyBuilder addToPath(String kind, String name) { - delegate.addAncestor(kind, name); + @Override + public KeyBuilder kind(String kind) { + super.kind(kind); return this; } - public KeyBuilder addToPath(String kind, long id) { - delegate.addAncestor(kind, id); + @Override + public KeyBuilder addAncestor(String kind, String name) { + super.addAncestor(kind, name); return this; } + @Override + public KeyBuilder addAncestor(String kind, long id) { + super.addAncestor(kind, id); + return this; + } + + @Override public KeyBuilder namespace(String namespace) { - delegate.namespace(namespace); + super.namespace(namespace); return this; } - /** - * Builds a key with a newly allocated id. - * @throws DatastoreServiceException if allocation failed. - */ - public Key allocateIdAndBuild() { - return service.allocateId(build()); + @Override + public KeyBuilder addAncestor(Ancestor... ancestor) { + super.addAncestor(ancestor); + return this; + } + + @Override + public KeyBuilder addAncestors(Iterable ancestors) { + super.addAncestors(ancestors); + return this; } public Key build(String name) { @@ -58,7 +73,11 @@ public Key build(long id) { .build(); } - public PartialKey build() { - return delegate.build(); + /** + * Builds a key with a newly allocated id. + * @throws DatastoreServiceException if allocation failed. + */ + public Key allocateIdAndBuild() { + return service.allocateId(build()); } } diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java index 5d77c333a54f..fb60ab20042f 100644 --- a/src/main/java/com/google/gcloud/datastore/NullValue.java +++ b/src/main/java/com/google/gcloud/datastore/NullValue.java @@ -51,7 +51,7 @@ public Builder set(Void value) { } public NullValue() { - this(new Builder().indexed(true)); + this(new Builder()); } NullValue(Builder builder) { diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 7827b66dcd33..8f7f0b085739 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -1,9 +1,9 @@ package com.google.gcloud.datastore; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; +import static com.google.gcloud.datastore.DatastoreServiceOptions.validateNamespace; import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; @@ -26,9 +26,9 @@ public class PartialKey extends Serializable { private final transient String dataset; private final transient String namespace; - private final transient ImmutableList path; + private final transient ImmutableList ancestors; - public static final class PathElement extends Serializable { + public static final class Ancestor extends Serializable { private static final long serialVersionUID = -7968078857690784595L; @@ -36,17 +36,17 @@ public static final class PathElement extends Serializable path = new LinkedList<>(); + private final List ancestors = new LinkedList<>(); - private static final String DEFAULT_NAMESPACE = ""; private static final int MAX_PATH = 100; public Builder(String dataset, String kind) { @@ -138,33 +137,33 @@ public Builder(String dataset, String kind) { public Builder(Key parent, String kind) { dataset = parent.dataset(); namespace = parent.namespace(); - path.addAll(parent.ancestors()); - path.add(parent.getLeaf()); + ancestors.addAll(parent.ancestors()); + ancestors.add(parent.getLeaf()); this.kind = kind; } public Builder addAncestor(String kind, long id) { checkArgument(id != 0, "id must not be equal to zero"); - return addAncestor(new PathElement(kind, id)); + return addAncestor(new Ancestor(kind, id)); } public Builder addAncestor(String kind, String name) { checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); checkArgument(name.length() <= 500, "name must not exceed 500 characters"); - return addAncestor(new PathElement(kind, name)); + return addAncestor(new Ancestor(kind, name)); } - public Builder addAncestor(PathElement... ancestor) { - Preconditions.checkState(path.size() + ancestor.length <= MAX_PATH, + public Builder addAncestor(Ancestor... ancestor) { + Preconditions.checkState(ancestors.size() + ancestor.length <= MAX_PATH, "path can have at most 100 elements"); - for (PathElement pathElement : ancestor) { - path.add(pathElement); + for (Ancestor pathElement : ancestor) { + ancestors.add(pathElement); } return this; } - public Builder addAncestors(Iterable ancestors) { - for (PathElement pathElement : ancestors) { + public Builder addAncestors(Iterable ancestors) { + for (Ancestor pathElement : ancestors) { addAncestor(pathElement); } return this; @@ -182,7 +181,7 @@ private String validateKind(String kind) { } public Builder clearPath() { - path.clear(); + ancestors.clear(); return this; } @@ -192,23 +191,23 @@ public Builder dataset(String dataset) { } public Builder namespace(String namespace) { - this.namespace = checkNotNull(namespace); + this.namespace = validateNamespace(namespace); return this; } public PartialKey build() { - PathElement leaf = new PathElement(kind); - ImmutableList pathList = - ImmutableList.builder().addAll(path).add(leaf).build(); + Ancestor leaf = new Ancestor(kind); + ImmutableList pathList = + ImmutableList.builder().addAll(ancestors).add(leaf).build(); return new PartialKey(dataset, namespace, pathList); } } - PartialKey(String dataset, String namespace, ImmutableList path) { + PartialKey(String dataset, String namespace, ImmutableList path) { checkState(!path.isEmpty(), "path must not be empty"); this.dataset = dataset; this.namespace = namespace; - this.path = path; + this.ancestors = path; } /** @@ -219,7 +218,7 @@ public String dataset() { } /** - * Returns the key's namespace. + * Returns the key's namespace or {@code null} if not provided. */ public String namespace() { return namespace; @@ -228,8 +227,8 @@ public String namespace() { /** * Returns an immutable list with the key's ancestors. */ - public List ancestors() { - return path.subList(0, path.size() - 1); + public List ancestors() { + return ancestors.subList(0, ancestors.size() - 1); } /** @@ -259,7 +258,7 @@ public Key toKey(long id) { @Override public int hashCode() { - return Objects.hash(dataset, namespace, path); + return Objects.hash(dataset, namespace, ancestors); } @Override @@ -270,26 +269,30 @@ public boolean equals(Object obj) { PartialKey other = (PartialKey) obj; return Objects.equals(dataset, other.dataset) && Objects.equals(namespace, other.namespace) - && Objects.equals(path, other.path); + && Objects.equals(ancestors, other.ancestors); } @Override protected DatastoreV1.Key toPb() { + return toPb(this).build(); + } + + static DatastoreV1.Key.Builder toPb(PartialKey key) { DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder(); - if (dataset != null) { - partitionIdPb.setDatasetId(dataset); + if (key.dataset != null) { + partitionIdPb.setDatasetId(key.dataset); } - if (namespace != null) { - partitionIdPb.setNamespace(namespace); + if (key.namespace != null) { + partitionIdPb.setNamespace(key.namespace); } if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { keyPb.setPartitionId(partitionIdPb.build()); } - for (PathElement pathEntry : path) { + for (Ancestor pathEntry : key.ancestors) { keyPb.addPathElement(pathEntry.toPb()); } - return keyPb.build(); + return keyPb; } @Override @@ -309,14 +312,14 @@ static PartialKey fromPb(DatastoreV1.Key keyPb) { namespace = partitionIdPb.getNamespace(); } } - ImmutableList.Builder pathBuilder = ImmutableList.builder(); + ImmutableList.Builder pathBuilder = ImmutableList.builder(); for (DatastoreV1.Key.PathElement pathElementPb : keyPb.getPathElementList()) { - pathBuilder.add(PathElement.fromPb(pathElementPb)); + pathBuilder.add(Ancestor.fromPb(pathElementPb)); } return new PartialKey(dataset, namespace, pathBuilder.build()); } - PathElement getLeaf() { - return path.get(path.size() - 1); + Ancestor getLeaf() { + return ancestors.get(ancestors.size() - 1); } } diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java new file mode 100644 index 000000000000..a5ec1bad4c05 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/package-info.java @@ -0,0 +1,31 @@ +/** + * A client to the Google Cloud Datastore. + * Typical usage would be: + * 

 {@code
+ * DatastoreServiceOptions options = new DatastoreServiceOptions.Builder().dataset(DATASET).build();
+ * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
+ * KeyBuilder keyBuilder = datastore.newKeyBuilder(kind);
+ * Key key = keyBuilder.build(keyName);
+ * Entity entity = datastore.get(key);
+ * if (entity == null) {
+ *   entity = new Entity.Builder(key)
+ *       .setProperty("name", new StringValue("John Do"))
+ *       .setProperty("age", new LongValue.Builder(100).indexed(false).build())
+ *       .setProperty("updated", new BooleanValue(false))
+ *       .build();
+ *   datastore.put(entity);
+ * } else {
+ *   BooleanValue updated = entity.property("updated");
+ *   if (!updated.get()) {
+ *     entity = entity.builder()
+ *         .setProperty("updated", new BooleanValue(true))
+ *         .removeProperty("old_property")
+ *         .setProperty("new_property", new DoubleValue(1.1))
+ *         .build();
+ *     datastore.update(entity);
+ *   }
+ * }
+ * }
+ * + */ +package com.google.gcloud.datastore; diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index fb00bb4dfa44..61debb6ead85 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -2,15 +2,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.google.api.services.datastore.client.LocalDevelopmentDatastore; -import com.google.api.services.datastore.client.LocalDevelopmentDatastoreException; - -import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -30,33 +27,43 @@ public class DatastoreServiceTest { private static final Key KEY1 = PARTIAL_KEY1.toKey("name"); private static final Key KEY2 = new Key.Builder(KEY1, KIND2, 1).build(); private static final Key KEY3 = KEY2.builder().name("bla").build(); + private static final PartialEntity PARTIAL_ENTITY1 = new PartialEntity.Builder(PARTIAL_KEY2) + .setProperty("str", STR_VALUE) + .setProperty("bool", BOOL_VALUE) + .build(); private static final Entity ENTITY1 = new Entity.Builder(KEY1) .setProperty("str", STR_VALUE) .setProperty("bool", BOOL_VALUE) + .setProperty("partial1", new PartialEntityValue(PARTIAL_ENTITY1)) .build(); private static final Entity ENTITY2 = new Entity.Builder(KEY2, ENTITY1) .removeProperty("str") .setProperty("null", NULL_VALUE) .build(); + private static final Entity ENTITY3 = new Entity.Builder(KEY3, ENTITY1) + .removeProperty("str") + .setProperty("null", NULL_VALUE) + .setProperty("partial2", new PartialEntityValue(ENTITY1)) + .build(); private DatastoreServiceOptions options; private DatastoreService datastore; - private LocalDevelopmentDatastore localDatastore; @Before - public void setUp() throws LocalDevelopmentDatastoreException { + public void setUp() { + // TODO(ozarov): document that this test depends on a local gcd running. + // gcd.sh start dataset1 + // reference: https://cloud.google.com/datastore/docs/tools/devserver + // Or even better, using a "GCE_HOME" param/env initiate and destroy the server + // before and after tests via ant or maven options = new DatastoreServiceOptions.Builder() .dataset(DATASET) - .host("http://localhost:8080/") + .host("http://localhost:8080") .build(); - datastore = DatastoreServiceFactory.Mode.TESTING.get(options); - localDatastore = (LocalDevelopmentDatastore) ((DatastoreServiceImpl) datastore).datastore(); - localDatastore.start("/usr/local/gcd-sdk", DATASET); - } - - @After - public void tearDown() throws LocalDevelopmentDatastoreException { - localDatastore.stop(); + datastore = DatastoreServiceFactory.getDefault(options); + // Prepare data for testing + datastore.delete(KEY1, KEY2, KEY3); + datastore.add(ENTITY1, ENTITY2); } @Test @@ -76,70 +83,139 @@ public void testNewBatchWriter() { @Test public void testAllocateId() { - fail("Not yet implemented"); + KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1); + PartialKey pk1 = keyBuilder.build(); + Key key1 = keyBuilder.allocateIdAndBuild(); + assertEquals(key1.dataset(), pk1.dataset()); + assertEquals(key1.namespace(), pk1.namespace()); + assertEquals(key1.ancestors(), pk1.ancestors()); + assertEquals(key1.kind(), pk1.kind()); + assertTrue(key1.hasId()); + assertFalse(key1.hasName()); + assertEquals(pk1.toKey(key1.id()), key1); + + Key key2 = datastore.allocateId(pk1); + assertNotEquals(key1, key2); + assertEquals(pk1.toKey(key2.id()), key2); + + Key key3 = datastore.allocateId(key1); + assertNotEquals(key1, key3); + assertEquals(pk1.toKey(key3.id()), key3); } @Test public void testAllocateIds() { - fail("Not yet implemented"); + KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1); + PartialKey key1 = keyBuilder.build(); + PartialKey key2 = keyBuilder.kind(KIND2).addAncestor(KIND1, 10).build(); + Iterator result = datastore.allocateIds(key1, key2); + Key key = result.next(); + assertEquals(key1.toKey(key.id()), key); + key = result.next(); + assertEquals(key2.toKey(key.id()), key); + assertFalse(result.hasNext()); } @Test - public void testAddAndGet() { - Entity entity = datastore.get(KEY1); + public void testGet() { + Entity entity = datastore.get(KEY3); assertNull(entity); - datastore.add(ENTITY1); - entity = datastore.get(KEY1); StringValue value1 = entity.property("str"); BooleanValue value2 = entity.property("bool"); + PartialEntityValue value3 = entity.property("partial1"); assertEquals(value1, STR_VALUE); assertEquals(value2, BOOL_VALUE); - assertEquals(2, entity.propertyNames().size()); + assertEquals(value3, new PartialEntityValue(PARTIAL_ENTITY1)); + assertEquals(3, entity.propertyNames().size()); assertTrue(entity.propertyNames().contains("str")); assertTrue(entity.propertyNames().contains("bool")); assertFalse(entity.hasProperty("bla")); } @Test - public void testAddAndGetArray() { - Iterator result = datastore.get(KEY1, KEY2); - assertNull(result.next()); - assertNull(result.next()); - assertFalse(result.hasNext()); - - populateDatastore(); - - result = datastore.get(KEY1, KEY1.builder().name("bla").build(), KEY2); + public void testGetArray() { + Iterator result = datastore.get(KEY1, KEY1.builder().name("bla").build(), KEY2); assertEquals(ENTITY1, result.next()); assertNull(result.next()); assertEquals(ENTITY2, result.next()); assertFalse(result.hasNext()); } - private void populateDatastore() { - datastore.add(ENTITY1, ENTITY2); - } - @Test public void testAdd() { - fail("Not yet implemented"); + Iterator keys = datastore.get(ENTITY1.key(), ENTITY3.key()); + assertEquals(ENTITY1, keys.next()); + assertNull(keys.next()); + assertFalse(keys.hasNext()); + + try { + datastore.add(ENTITY1); + } catch (DatastoreServiceException expected) { + // expected; + } + datastore.add(ENTITY3); + assertEquals(ENTITY3, datastore.get(ENTITY3.key())); } @Test public void testUpdate() { - fail("Not yet implemented"); + Iterator keys = datastore.get(ENTITY1.key(), ENTITY3.key()); + assertEquals(ENTITY1, keys.next()); + assertNull(keys.next()); + assertFalse(keys.hasNext()); + + try { + datastore.update(ENTITY3); + } catch (DatastoreServiceException expected) { + // expected; + } + datastore.add(ENTITY3); + assertEquals(ENTITY3, datastore.get(ENTITY3.key())); + Entity entity3 = ENTITY3.builder() + .clearProperties() + .setProperty("bla", new NullValue()) + .build(); + assertNotEquals(ENTITY3, entity3); + datastore.update(entity3); + assertEquals(entity3, datastore.get(ENTITY3.key())); } @Test public void testPut() { - fail("Not yet implemented"); + Iterator keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()); + assertEquals(ENTITY1, keys.next()); + assertEquals(ENTITY2, keys.next()); + assertNull(keys.next()); + assertFalse(keys.hasNext()); + + Entity entity2 = ENTITY2.builder() + .clearProperties() + .setProperty("bla", new NullValue()) + .build(); + assertNotEquals(ENTITY2, entity2); + datastore.put(ENTITY3, ENTITY1, entity2); + keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()); + assertEquals(ENTITY1, keys.next()); + assertEquals(entity2, keys.next()); + assertEquals(ENTITY3, keys.next()); + assertFalse(keys.hasNext()); } @Test public void testDelete() { - fail("Not yet implemented"); + Iterator keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()); + assertEquals(ENTITY1, keys.next()); + assertEquals(ENTITY2, keys.next()); + assertNull(keys.next()); + assertFalse(keys.hasNext()); + datastore.delete(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()); + keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()); + assertNull(keys.next()); + assertNull(keys.next()); + assertNull(keys.next()); + assertFalse(keys.hasNext()); } @Test From 71a3478d21e39d21a933753ad7e03aea679613e2 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 4 Dec 2014 18:55:04 -0800 Subject: [PATCH 029/732] adding batch and transaction --- .../gcloud/datastore/BatchWriteOption.java | 17 ++- .../google/gcloud/datastore/BatchWriter.java | 14 ++ .../gcloud/datastore/BatchWriterImpl.java | 139 ++++++++++++++++++ .../gcloud/datastore/DatastoreService.java | 8 +- .../datastore/DatastoreServiceImpl.java | 22 ++- .../gcloud/datastore/DatastoreWriter.java | 4 +- .../gcloud/datastore/TransactionImpl.java | 64 ++++++++ .../gcloud/datastore/TransactionOption.java | 49 +++--- 8 files changed, 276 insertions(+), 41 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java create mode 100644 src/main/java/com/google/gcloud/datastore/TransactionImpl.java diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java index cda7bf7be16f..b860b118f2fa 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java @@ -6,10 +6,6 @@ public abstract class BatchWriteOption implements Serializable { private static final long serialVersionUID = -3932758377282659839L; - BatchWriteOption() { - // package protected - } - public static final class ForceWrites extends BatchWriteOption { private static final long serialVersionUID = 2555054296046232799L; @@ -20,11 +16,22 @@ public ForceWrites(boolean force) { this.force = force; } - public boolean isForce() { + public boolean force() { return force; } + + @Override + void apply(BatchWriterImpl batchWriter) { + batchWriter.apply(this); + } } + BatchWriteOption() { + // package protected + } + + abstract void apply(BatchWriterImpl batchWriter); + public static ForceWrites forceWrites() { return new ForceWrites(true); } diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java index c8e1056b21d9..889ff425a1ab 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java @@ -2,6 +2,20 @@ public interface BatchWriter extends DatastoreWriter { + /** + * {@inheritDoc} + * @throws DatastoreServiceException if a given entity already added to this batch + */ + @Override + void add(Entity... entity); + + /** + * {@inheritDoc} + * @throws DatastoreServiceException if an entity is marked for deletion in this batch + */ + @Override + void update(Entity... entity); + /** * Submit the batch to the Datastore. * diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java new file mode 100644 index 000000000000..f80c0e17af2f --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java @@ -0,0 +1,139 @@ +package com.google.gcloud.datastore; + +import static com.google.gcloud.datastore.DatastoreServiceException.Code.FAILED_PRECONDITION; + +import com.google.api.services.datastore.DatastoreV1; + +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; + +class BatchWriterImpl implements BatchWriter { + + private final LinkedHashMap toAdd = new LinkedHashMap<>(); + private final LinkedHashMap toUpdate = new LinkedHashMap<>(); + private final LinkedHashMap toPut = new LinkedHashMap<>(); + private final LinkedHashSet toDelete = new LinkedHashSet<>(); + private final DatastoreServiceImpl datastore; + + private boolean force; + protected boolean isValid = true; + + BatchWriterImpl(DatastoreServiceImpl datastore, BatchWriteOption... options) { + this.datastore = datastore; + force = datastore.getOptions().force(); + for (BatchWriteOption option : options) { + option.apply(this); + } + } + + // Apply all valid options + + void apply(BatchWriteOption.ForceWrites forceOptions) { + this.force = forceOptions.force(); + } + + void apply(BatchWriteOption other) { + // dont care + } + + //////////////////// + + DatastoreServiceException newBatchFailure(Entity entity, String msg) { + isValid = false; + return new DatastoreServiceException(FAILED_PRECONDITION, + new RuntimeException("Entity with the key " + entity.key() + " " + msg)); + } + + protected void checkValid() { + if (!isValid) { + throw new DatastoreServiceException(FAILED_PRECONDITION, + new RuntimeException("BatchWriter is in an invalid state")); + } + } + + @Override + public void add(Entity... entities) { + checkValid(); + for (Entity entity : entities) { + Key key = entity.key(); + if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) { + throw newBatchFailure(entity, "was already added or updated in this batch"); + } + if (toDelete.remove(key)) { + toPut.put(key, entity); + } else { + toAdd.put(key, entity); + } + } + } + + @Override + public void update(Entity... entities) { + checkValid(); + for (Entity entity : entities) { + Key key = entity.key(); + if (toDelete.contains(key)) { + throw newBatchFailure(entity, "was alredy deleted in this batch"); + } + if (toAdd.remove(key) != null || toPut.containsKey(key)) { + toPut.put(key, entity); + } else { + toUpdate.put(key, entity); + } + } + } + + @Override + public void put(Entity... entities) { + checkValid(); + for (Entity entity : entities) { + Key key = entity.key(); + toAdd.remove(key); + toUpdate.remove(key); + toDelete.remove(key); + toPut.put(key, entity); + } + } + + @Override + public void delete(Key... keys) { + checkValid(); + for (Key key : keys) { + toAdd.remove(key); + toUpdate.remove(key); + toPut.remove(key); + toDelete.add(key); + } + } + + @Override + public void submit() { + checkValid(); + DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); + for (Entity entity : toAdd.values()) { + mutationPb.addInsert(entity.toPb()); + } + for (Entity entity : toUpdate.values()) { + mutationPb.addUpdate(entity.toPb()); + } + for (Entity entity : toPut.values()) { + mutationPb.addUpsert(entity.toPb()); + } + for (Key key : toDelete) { + mutationPb.addDelete(key.toPb()); + } + if (force) { + mutationPb.setForce(force); + } + DatastoreV1.CommitRequest.Builder requestPb = newCommitRequest(); + requestPb.setMutation(mutationPb); + datastore.comitMutation(requestPb); + isValid = false; + } + + protected DatastoreV1.CommitRequest.Builder newCommitRequest() { + DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); + requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL); + return requestPb; + } +} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 37fb52ad701f..8a0472e4e995 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -49,28 +49,28 @@ public interface DatastoreService extends DatastoreReader, DatastoreWriter { Iterator allocateIds(PartialKey... key); /** - * @see DatastoreWriter#add(Entity...) + * {@inheritDoc} * @throws DatastoreServiceExcepiton upon failure */ @Override void add(Entity... entity); /** - * @see DatastoreWriter#update(Entity...) + * {@inheritDoc} * @throws DatastoreServiceExcepiton upon failure */ @Override void update(Entity... entity); /** - * @see DatastoreWriter#put(Entity...) + * {@inheritDoc} * @throws DatastoreServiceExcepiton upon failure */ @Override void put(Entity... entity); /** - * @see DatastoreWriter#delete(Key...) + * {@inheritDoc} * @throws DatastoreServiceExcepiton upon failure */ @Override diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 4dcae9612568..6fdd57f622ce 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -1,9 +1,11 @@ package com.google.gcloud.datastore; import com.google.api.services.datastore.DatastoreV1; +import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse; import com.google.api.services.datastore.client.Datastore; import com.google.api.services.datastore.client.DatastoreException; import com.google.common.collect.AbstractIterator; +import com.google.protobuf.ByteString; import java.util.HashMap; import java.util.Iterator; @@ -27,14 +29,21 @@ public DatastoreServiceOptions getOptions() { @Override public Transaction newTransaction(TransactionOption... transactionOption) { - // TODO Auto-generated method stub - return null; + return new TransactionImpl(this, transactionOption); + } + + ByteString requestTransactionId(DatastoreV1.BeginTransactionRequest.Builder requestPb) { + try { + BeginTransactionResponse responsePb = datastore.beginTransaction(requestPb.build()); + return responsePb.getTransaction(); + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndPropagate(e); + } } @Override public BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption) { - // TODO Auto-generated method stub - return null; + return new BatchWriterImpl(this, batchWriteOption); } @Override @@ -123,7 +132,10 @@ private void comitMutation(DatastoreV1.Mutation.Builder mutationPb) { } DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL); - requestPb.setMutation(mutationPb.build()); + requestPb.setMutation(mutationPb); + } + + void comitMutation(DatastoreV1.CommitRequest.Builder requestPb) { try { datastore.commit(requestPb.build()); } catch (DatastoreException e) { diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java index cfb864744099..5c87a49424ba 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java @@ -7,13 +7,13 @@ public interface DatastoreWriter { /** * A Datastore add operation. - * The operation will fail if an entity already exists. + * The operation will fail if an entity with the same key already exists. */ void add(Entity... entity); /** * A Datastore update operation. - * The operation will fail if an entity does not already exist. + * The operation will fail if an entity with the same key does not already exist. */ void update(Entity... entity); diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java new file mode 100644 index 000000000000..61db951e343c --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java @@ -0,0 +1,64 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.gcloud.datastore.TransactionOption.IsolationLevel; +import com.google.protobuf.ByteString; + +import java.util.Iterator; + +public final class TransactionImpl extends BatchWriterImpl implements Transaction { + + private final ByteString transaction; + private IsolationLevel isolationLevel; + + TransactionImpl(DatastoreServiceImpl datastore, TransactionOption... options) { + super(datastore, options); + DatastoreV1.BeginTransactionRequest.Builder requestPb = + DatastoreV1.BeginTransactionRequest.newBuilder(); + if (isolationLevel != null) { + requestPb.setIsolationLevel(isolationLevel.level().toPb()); + } + transaction = datastore.requestTransactionId(requestPb); + } + + void apply(IsolationLevel isolationLevel) { + // TODO(ozarov): validate that this concept actually works!!! + this.isolationLevel = isolationLevel; + } + + @Override + public Entity get(Key key) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Iterator get(Key... key) { + // TODO Auto-generated method stub + return null; + } + + @Override + public QueryResult runQuery(Query query) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void commit() { + submit(); + } + + @Override + public void rollback() { + isValid = false; + } + + @Override + protected DatastoreV1.CommitRequest.Builder newCommitRequest() { + DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); + requestPb.setMode(DatastoreV1.CommitRequest.Mode.TRANSACTIONAL); + requestPb.setTransaction(transaction); + return requestPb; + } +} diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/src/main/java/com/google/gcloud/datastore/TransactionOption.java index 50bd2aac2869..56e2df445b17 100644 --- a/src/main/java/com/google/gcloud/datastore/TransactionOption.java +++ b/src/main/java/com/google/gcloud/datastore/TransactionOption.java @@ -1,15 +1,11 @@ package com.google.gcloud.datastore; -import java.io.Serializable; +import com.google.api.services.datastore.DatastoreV1; -public abstract class TransactionOption implements Serializable { - - private static final long serialVersionUID = -1862234444015690375L; +public abstract class TransactionOption extends BatchWriteOption { - TransactionOption() { - // package protected - } + private static final long serialVersionUID = -1862234444015690375L; public static final class IsolationLevel extends TransactionOption { @@ -18,31 +14,38 @@ public static final class IsolationLevel extends TransactionOption { private final Level level; public enum Level { - SERIALIZABLE, SNAPSHOT; + + SERIALIZABLE(DatastoreV1.BeginTransactionRequest.IsolationLevel.SERIALIZABLE), + SNAPSHOT(DatastoreV1.BeginTransactionRequest.IsolationLevel.SNAPSHOT); + + private final DatastoreV1.BeginTransactionRequest.IsolationLevel levelPb; + + Level(DatastoreV1.BeginTransactionRequest.IsolationLevel levelPb) { + this.levelPb = levelPb; + } + + DatastoreV1.BeginTransactionRequest.IsolationLevel toPb() { + return levelPb; + } } public IsolationLevel(Level level) { this.level = level; } - public Level getLevel() { + + public Level level() { return level; } - } - - public static final class ForceWrites extends TransactionOption { - - private static final long serialVersionUID = 7448106703678852594L; - private final boolean force; - - public ForceWrites(boolean force) { - this.force = force; + @Override + void apply(BatchWriterImpl batchWriter) { + batchWriter.apply(this); } + } - public boolean isForce() { - return force; - } + TransactionOption() { + // package protected } public static IsolationLevel serializable() { @@ -52,8 +55,4 @@ public static IsolationLevel serializable() { public static IsolationLevel snapshot() { return new IsolationLevel(IsolationLevel.Level.SNAPSHOT); } - - public static ForceWrites forceWrites() { - return new ForceWrites(true); - } } From 4a7eaf15e8ab0ab6f5e408c53ecb15feb5fc807e Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 5 Dec 2014 12:22:43 -0800 Subject: [PATCH 030/732] more documenation and complete batch and transaction (no tests yet) --- .../gcloud/datastore/BatchWriteOption.java | 20 +- .../gcloud/datastore/BatchWriterImpl.java | 54 +++--- .../gcloud/datastore/DatastoreReader.java | 2 +- .../gcloud/datastore/DatastoreService.java | 2 +- .../datastore/DatastoreServiceException.java | 17 +- .../datastore/DatastoreServiceImpl.java | 180 +++++++++++------- .../google/gcloud/datastore/Transaction.java | 36 ++++ .../gcloud/datastore/TransactionImpl.java | 46 +++-- .../gcloud/datastore/TransactionOption.java | 5 - .../datastore/DatastoreServiceTest.java | 35 +++- 10 files changed, 260 insertions(+), 137 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java index b860b118f2fa..2fea6e95d893 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java @@ -1,6 +1,9 @@ package com.google.gcloud.datastore; +import com.google.common.collect.ImmutableMap; + import java.io.Serializable; +import java.util.Map; public abstract class BatchWriteOption implements Serializable { @@ -19,20 +22,23 @@ public ForceWrites(boolean force) { public boolean force() { return force; } - - @Override - void apply(BatchWriterImpl batchWriter) { - batchWriter.apply(this); - } } BatchWriteOption() { // package protected } - abstract void apply(BatchWriterImpl batchWriter); - public static ForceWrites forceWrites() { return new ForceWrites(true); } + + static Map, BatchWriteOption> asImmutableMap( + BatchWriteOption... options) { + ImmutableMap.Builder, BatchWriteOption> builder = + ImmutableMap.builder(); + for (BatchWriteOption option : options) { + builder.put(option.getClass(), option); + } + return builder.build(); + } } diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java index f80c0e17af2f..89fbc60742e2 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java @@ -1,11 +1,13 @@ package com.google.gcloud.datastore; -import static com.google.gcloud.datastore.DatastoreServiceException.Code.FAILED_PRECONDITION; +import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest; import com.google.api.services.datastore.DatastoreV1; +import com.google.gcloud.datastore.BatchWriteOption.ForceWrites; import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.Map; class BatchWriterImpl implements BatchWriter { @@ -13,51 +15,40 @@ class BatchWriterImpl implements BatchWriter { private final LinkedHashMap toUpdate = new LinkedHashMap<>(); private final LinkedHashMap toPut = new LinkedHashMap<>(); private final LinkedHashSet toDelete = new LinkedHashSet<>(); - private final DatastoreServiceImpl datastore; + private final boolean force; + protected final DatastoreServiceImpl datastore; - private boolean force; - protected boolean isValid = true; + private boolean wasSubmitted = false; BatchWriterImpl(DatastoreServiceImpl datastore, BatchWriteOption... options) { this.datastore = datastore; - force = datastore.getOptions().force(); - for (BatchWriteOption option : options) { - option.apply(this); + Map, BatchWriteOption> optionsMap = + BatchWriteOption.asImmutableMap(options); + if (optionsMap.containsKey(ForceWrites.class)) { + force = ((ForceWrites) optionsMap.get(ForceWrites.class)).force(); + } else { + force = datastore.getOptions().force(); } } - // Apply all valid options - - void apply(BatchWriteOption.ForceWrites forceOptions) { - this.force = forceOptions.force(); - } - - void apply(BatchWriteOption other) { - // dont care - } - - //////////////////// - - DatastoreServiceException newBatchFailure(Entity entity, String msg) { - isValid = false; - return new DatastoreServiceException(FAILED_PRECONDITION, - new RuntimeException("Entity with the key " + entity.key() + " " + msg)); - } - protected void checkValid() { - if (!isValid) { - throw new DatastoreServiceException(FAILED_PRECONDITION, - new RuntimeException("BatchWriter is in an invalid state")); + if (wasSubmitted) { + throwInvalidRequest(getName() + " was already submitted"); } } + protected String getName() { + return "batch"; + } + @Override public void add(Entity... entities) { checkValid(); for (Entity entity : entities) { Key key = entity.key(); if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) { - throw newBatchFailure(entity, "was already added or updated in this batch"); + throw throwInvalidRequest("Entity with the key %s was already added or updated in this " + + getName(), entity.key()); } if (toDelete.remove(key)) { toPut.put(key, entity); @@ -73,7 +64,8 @@ public void update(Entity... entities) { for (Entity entity : entities) { Key key = entity.key(); if (toDelete.contains(key)) { - throw newBatchFailure(entity, "was alredy deleted in this batch"); + throw throwInvalidRequest( + "Entity with the key %s was already deleted in this " + getName(), entity.key()); } if (toAdd.remove(key) != null || toPut.containsKey(key)) { toPut.put(key, entity); @@ -128,7 +120,7 @@ public void submit() { DatastoreV1.CommitRequest.Builder requestPb = newCommitRequest(); requestPb.setMutation(mutationPb); datastore.comitMutation(requestPb); - isValid = false; + wasSubmitted = true; } protected DatastoreV1.CommitRequest.Builder newCommitRequest() { diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java index 32b4dbd73461..ac2e048eff0f 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java @@ -20,7 +20,7 @@ public interface DatastoreReader { * * @throws DatastoreServiceException upon failure. */ - Iterator get(Key... key); + Iterator get(Key key, Key... others); /** * Submit a {@link Query} and returns its result. diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java index 8a0472e4e995..b7633c49cd10 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java @@ -46,7 +46,7 @@ public interface DatastoreService extends DatastoreReader, DatastoreWriter { * @see #allocateId(PartialKey) * @throws DatastoreServiceExcepiton upon failure */ - Iterator allocateIds(PartialKey... key); + Iterator allocateId(PartialKey key, PartialKey... others); /** * {@inheritDoc} diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java index 912efe4a4692..94c16966f1fb 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java @@ -69,9 +69,22 @@ public Code code() { * Translate DatastoreException to DatastoreServiceException based on their * HTTP error codes. This method will always throw a new DatastoreServiceException. * - * @throws DatastoreServiceException for every given DatastoreException + * @throws DatastoreServiceException every time */ - static DatastoreServiceException translateAndPropagate(DatastoreException exception) { + static DatastoreServiceException translateAndThrow(DatastoreException exception) { throw firstNonNull(HTTP_TO_CODE.get(exception.getCode()), Code.UNKNOWN).translate(exception); } + + /** + * Throw a DatastoreServiceException with {@code FAILED_PRECONDITION} code and the {@code msg} + * in a nested exception. + * + * @throws DatastoreServiceException every time + */ + static DatastoreServiceException throwInvalidRequest(String msg, Object... params) { + if (params.length > 0) { + msg = String.format(msg, params); + } + throw new DatastoreServiceException(Code.FAILED_PRECONDITION, new RuntimeException(msg)); + } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 6fdd57f622ce..69819b1c73f0 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -7,13 +7,19 @@ import com.google.common.collect.AbstractIterator; import com.google.protobuf.ByteString; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; final class DatastoreServiceImpl implements DatastoreService { + static final Key[] EMPTY_KEY_ARRAY = {}; + static final PartialKey[] EMPTY_PARTIAL_KEY_ARRAY = {}; + private final DatastoreServiceOptions options; private final Datastore datastore; @@ -28,17 +34,8 @@ public DatastoreServiceOptions getOptions() { } @Override - public Transaction newTransaction(TransactionOption... transactionOption) { - return new TransactionImpl(this, transactionOption); - } - - ByteString requestTransactionId(DatastoreV1.BeginTransactionRequest.Builder requestPb) { - try { - BeginTransactionResponse responsePb = datastore.beginTransaction(requestPb.build()); - return responsePb.getTransaction(); - } catch (DatastoreException e) { - throw DatastoreServiceException.translateAndPropagate(e); - } + public KeyBuilder newKeyBuilder(String kind) { + return new KeyBuilder(this, kind); } @Override @@ -46,23 +43,29 @@ public BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption) { return new BatchWriterImpl(this, batchWriteOption); } + @Override + public Transaction newTransaction(TransactionOption... transactionOption) { + return new TransactionImpl(this, transactionOption); + } + + @Override + public QueryResult runQuery(Query query) { + // TODO To implement + throw new RuntimeException("Not implemented yet"); + } + + @Override public Key allocateId(PartialKey key) { - return allocateIds(key).next(); + return allocateId(key, EMPTY_PARTIAL_KEY_ARRAY).next(); } @Override - public Iterator allocateIds(PartialKey... key) { + public Iterator allocateId(PartialKey key, PartialKey... others) { DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder(); - for (PartialKey k : key) { - if (k.getLeaf().nameOrId() != null) { - // if key is full remove the id or name part - k = new PartialKey.Builder(k.dataset(), k.kind()) - .namespace(k.namespace()) - .addAncestors(k.ancestors()) - .build(); - } - requestPb.addKey(k.toPb()); + requestPb.addKey(trimNameOrId(key).toPb()); + for (PartialKey other : others) { + requestPb.addKey(trimNameOrId(other).toPb()); } // TODO(ozarov): will need to populate "force" after b/18594027 is fixed. try { @@ -79,19 +82,39 @@ protected Key computeNext() { } }; } catch (DatastoreException e) { - throw DatastoreServiceException.translateAndPropagate(e); + throw DatastoreServiceException.translateAndThrow(e); } } + private PartialKey trimNameOrId(PartialKey key) { + if (key.getLeaf().nameOrId() == null) { + return key; + } + return new PartialKey.Builder(key.dataset(), key.kind()) + .namespace(key.namespace()) + .addAncestors(key.ancestors()) + .build(); + } + @Override public Entity get(Key key) { - return get(new Key[]{key}).next(); + return get(key, EMPTY_KEY_ARRAY).next(); } @Override - public Iterator get(final Key... key) { + public Iterator get(Key key, Key... others) { + return get(null, key, others); + } + + Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key key, final Key... others) { DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder(); - for (Key k : key) { + if (readOptionsPb != null) { + requestPb.setReadOptions(readOptionsPb); + } + LinkedHashSet dedupKeys = new LinkedHashSet<>(); + dedupKeys.add(key); + dedupKeys.addAll(Arrays.asList(others)); + for (Key k : dedupKeys) { requestPb.addKey(k.toPb()); } try { @@ -102,86 +125,109 @@ public Iterator get(final Key... key) { result.put(entity.key(), entity); } return new AbstractIterator() { - int index; + int index = -2; @Override protected Entity computeNext() { - if (index < key.length) { - return result.get(key[index++]); + ++index; + if (index < 0) { + return result.get(key); + } + if (index < others.length) { + return result.get(others[index]); } return endOfData(); } }; } catch (DatastoreException e) { - throw DatastoreServiceException.translateAndPropagate(e); + throw DatastoreServiceException.translateAndThrow(e); } } @Override - public void add(Entity... entity) { + public void add(Entity... entities) { DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); - for (Entity e : entity) { - mutationPb.addInsert(e.toPb()); + LinkedHashSet keys = new LinkedHashSet<>(); + for (Entity entity : entities) { + if (!keys.add(entity.key())) { + throw DatastoreServiceException.throwInvalidRequest( + "Duplicate entity with the key %s", entity.key()); + } + mutationPb.addInsert(entity.toPb()); } comitMutation(mutationPb); } - private void comitMutation(DatastoreV1.Mutation.Builder mutationPb) { - if (options.force()) { - mutationPb.setForce(true); - } - DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); - requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL); - requestPb.setMutation(mutationPb); - } - - void comitMutation(DatastoreV1.CommitRequest.Builder requestPb) { - try { - datastore.commit(requestPb.build()); - } catch (DatastoreException e) { - throw DatastoreServiceException.translateAndPropagate(e); - } - } - @Override - public void update(Entity... entity) { + public void update(Entity... entities) { DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); - for (Entity e : entity) { - mutationPb.addUpdate(e.toPb()); + LinkedHashMap dedupEntities = new LinkedHashMap<>(); + for (Entity entity : entities) { + dedupEntities.put(entity.key(), entity); + } + for (Entity entity : dedupEntities.values()) { + mutationPb.addUpdate(entity.toPb()); } comitMutation(mutationPb); } @Override - public void put(Entity... entity) { + public void put(Entity... entities) { DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); - for (Entity e : entity) { + LinkedHashMap dedupEntities = new LinkedHashMap<>(); + for (Entity entity : entities) { + dedupEntities.put(entity.key(), entity); + } + for (Entity e : dedupEntities.values()) { mutationPb.addUpsert(e.toPb()); } comitMutation(mutationPb); } @Override - public void delete(Key... key) { + public void delete(Key... keys) { DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); - for (Key k : key) { - mutationPb.addDelete(k.toPb()); + LinkedHashSet dedupKeys = new LinkedHashSet<>(Arrays.asList(keys)); + for (Key key : dedupKeys) { + mutationPb.addDelete(key.toPb()); } comitMutation(mutationPb); } - @Override - public KeyBuilder newKeyBuilder(String kind) { - return new KeyBuilder(this, kind); + private void comitMutation(DatastoreV1.Mutation.Builder mutationPb) { + if (options.force()) { + mutationPb.setForce(true); + } + DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder(); + requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL); + requestPb.setMutation(mutationPb); + comitMutation(requestPb); } - @Override - public QueryResult runQuery(Query query) { - // TODO Auto-generated method stub - return null; + void comitMutation(DatastoreV1.CommitRequest.Builder requestPb) { + try { + datastore.commit(requestPb.build()); + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndThrow(e); + } } - Datastore datastore() { - return datastore; + ByteString requestTransactionId(DatastoreV1.BeginTransactionRequest.Builder requestPb) { + try { + BeginTransactionResponse responsePb = datastore.beginTransaction(requestPb.build()); + return responsePb.getTransaction(); + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndThrow(e); + } + } + + void rollbackTransaction(ByteString transaction) { + DatastoreV1.RollbackRequest.Builder requestPb = DatastoreV1.RollbackRequest.newBuilder(); + requestPb.setTransaction(transaction); + try { + datastore.rollback(requestPb.build()); + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndThrow(e); + } } } diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java index fd916960d78a..4138b8409d98 100644 --- a/src/main/java/com/google/gcloud/datastore/Transaction.java +++ b/src/main/java/com/google/gcloud/datastore/Transaction.java @@ -1,5 +1,7 @@ package com.google.gcloud.datastore; +import java.util.Iterator; + /** * A Google cloud datastore transaction. * @@ -7,6 +9,40 @@ */ public interface Transaction extends DatastoreReader, DatastoreWriter { + /** + * {@inheritDoc} + * The requested entity will be part of this Datastore transaction (so a commit is guaranteed + * to fail if entity was changed by others after it was seen by this transaction) but any + * write changes in this transaction will not be reflected by the returned entity. + * + * @throws DatastoreServiceException upon failure. + */ + @Override + Entity get(Key key); + + /** + * {@inheritDoc} + * The requested entities will be part of this Datastore transaction (so a commit is guaranteed + * to fail if any of the entities was changed by others after they were seen by this transaction) + * but any write changes in this transaction will not be reflected by the returned entities. + * + * @throws DatastoreServiceException upon failure. + */ + @Override + Iterator get(Key key, Key... others); + + /** + * {@inheritDoc} + * The entities returned by the result of this query will be part of this Datastore transaction + * (so a commit is guaranteed to fail if any of the entities was changed by others after the + * query was performed) but any write changes in this transaction will not be reflected by + * the result. + * + * @throws DatastoreServiceException upon failure. + */ + @Override + QueryResult runQuery(Query query); + /** * Commit the transaction. * diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java index 61db951e343c..17fe3e1339fc 100644 --- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java +++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java @@ -1,47 +1,50 @@ package com.google.gcloud.datastore; +import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest; + import com.google.api.services.datastore.DatastoreV1; import com.google.gcloud.datastore.TransactionOption.IsolationLevel; import com.google.protobuf.ByteString; import java.util.Iterator; +import java.util.Map; public final class TransactionImpl extends BatchWriterImpl implements Transaction { private final ByteString transaction; - private IsolationLevel isolationLevel; + private boolean wasRolledback; TransactionImpl(DatastoreServiceImpl datastore, TransactionOption... options) { super(datastore, options); DatastoreV1.BeginTransactionRequest.Builder requestPb = DatastoreV1.BeginTransactionRequest.newBuilder(); + Map, BatchWriteOption> optionsMap = + BatchWriteOption.asImmutableMap(options); + IsolationLevel isolationLevel = (IsolationLevel) optionsMap.get(IsolationLevel.class); if (isolationLevel != null) { requestPb.setIsolationLevel(isolationLevel.level().toPb()); } transaction = datastore.requestTransactionId(requestPb); } - void apply(IsolationLevel isolationLevel) { - // TODO(ozarov): validate that this concept actually works!!! - this.isolationLevel = isolationLevel; - } - @Override public Entity get(Key key) { - // TODO Auto-generated method stub - return null; + return get(key, DatastoreServiceImpl.EMPTY_KEY_ARRAY).next(); } @Override - public Iterator get(Key... key) { - // TODO Auto-generated method stub - return null; + public Iterator get(Key key, Key... others) { + checkValid(); + DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder(); + readOptionsPb.setTransaction(transaction); + return datastore.get(readOptionsPb.build(), key, others); } @Override public QueryResult runQuery(Query query) { - // TODO Auto-generated method stub - return null; + checkValid(); + // TODO To implement + throw new RuntimeException("Not implemented yet"); } @Override @@ -51,7 +54,22 @@ public void commit() { @Override public void rollback() { - isValid = false; + super.checkValid(); + datastore.rollbackTransaction(transaction); + wasRolledback = true; + } + + @Override + protected String getName() { + return "transaction"; + } + + @Override + protected void checkValid() { + super.checkValid(); + if (wasRolledback) { + throwInvalidRequest(getName() + " was already rolledback"); + } } @Override diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/src/main/java/com/google/gcloud/datastore/TransactionOption.java index 56e2df445b17..0471929e7fb1 100644 --- a/src/main/java/com/google/gcloud/datastore/TransactionOption.java +++ b/src/main/java/com/google/gcloud/datastore/TransactionOption.java @@ -37,11 +37,6 @@ public IsolationLevel(Level level) { public Level level() { return level; } - - @Override - void apply(BatchWriterImpl batchWriter) { - batchWriter.apply(this); - } } TransactionOption() { diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 61debb6ead85..2c3914d2a4f6 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -11,7 +11,9 @@ import org.junit.Before; import org.junit.Test; +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; public class DatastoreServiceTest { @@ -78,6 +80,12 @@ public void testNewTransaction() { @Test public void testNewBatchWriter() { + + fail("Not yet implemented"); + } + + @Test + public void testRunQuery() { fail("Not yet implemented"); } @@ -104,16 +112,25 @@ public void testAllocateId() { } @Test - public void testAllocateIds() { + public void testAllocateIdArray() { KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1); - PartialKey key1 = keyBuilder.build(); - PartialKey key2 = keyBuilder.kind(KIND2).addAncestor(KIND1, 10).build(); - Iterator result = datastore.allocateIds(key1, key2); - Key key = result.next(); - assertEquals(key1.toKey(key.id()), key); - key = result.next(); - assertEquals(key2.toKey(key.id()), key); - assertFalse(result.hasNext()); + PartialKey pKey1 = keyBuilder.build(); + PartialKey pKey2 = keyBuilder.kind(KIND2).addAncestor(KIND1, 10).build(); + Key key3 = keyBuilder.build("name"); + Key key4 = keyBuilder.build(1); + Iterator result = datastore.allocateId(pKey1, pKey2, key3, key4, pKey1, key3); + Map map = new HashMap<>(); + int count = 0; + while (result.hasNext()) { + map.put(++count, result.next()); + } + assertEquals(6, map.size()); + assertEquals(pKey1.toKey(map.get(1).id()), map.get(1)); + assertEquals(pKey1.toKey(map.get(5).id()), map.get(5)); + assertEquals(pKey2.toKey(map.get(2).id()), map.get(2)); + assertEquals(key3.builder().id(map.get(3).id()).build(), map.get(3)); + assertEquals(key3.builder().id(map.get(6).id()).build(), map.get(6)); + assertEquals(key4.builder().id(map.get(4).id()).build(), map.get(4)); } @Test From 1222d1e35a8add26788843c101530e30c74755c3 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 5 Dec 2014 12:45:59 -0800 Subject: [PATCH 031/732] Added some tests for batchwrite --- .../google/gcloud/datastore/PartialKey.java | 4 +- .../datastore/DatastoreServiceTest.java | 62 ++++++++++++++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 8f7f0b085739..d51cd19cd621 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -242,14 +242,14 @@ public Builder builder() { return new Builder(dataset(), kind()).namespace(namespace()).addAncestors(ancestors()); } - public Key toKey(String name) { + public Key newKey(String name) { return new Key.Builder(dataset(), kind(), name) .namespace(namespace()) .addAncestors(ancestors()) .build(); } - public Key toKey(long id) { + public Key newKey(long id) { return new Key.Builder(dataset(), kind(), id) .namespace(namespace()) .addAncestors(ancestors()) diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 2c3914d2a4f6..4e9c23b421c1 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -26,7 +26,7 @@ public class DatastoreServiceTest { new BooleanValue.Builder(false).indexed(false).build(); private static final PartialKey PARTIAL_KEY1 = new PartialKey.Builder(DATASET, KIND1).build(); private static final PartialKey PARTIAL_KEY2 = new PartialKey.Builder(DATASET, KIND2).build(); - private static final Key KEY1 = PARTIAL_KEY1.toKey("name"); + private static final Key KEY1 = PARTIAL_KEY1.newKey("name"); private static final Key KEY2 = new Key.Builder(KEY1, KIND2, 1).build(); private static final Key KEY3 = KEY2.builder().name("bla").build(); private static final PartialEntity PARTIAL_ENTITY1 = new PartialEntity.Builder(PARTIAL_KEY2) @@ -74,14 +74,58 @@ public void testGetOptions() { } @Test - public void testNewTransaction() { + public void testNewTransactionCommit() { + fail("Not yet implemented"); + } + + @Test + public void testNewTransactionRollback() { fail("Not yet implemented"); } @Test public void testNewBatchWriter() { + BatchWriter batchWriter = datastore.newBatchWriter(); + Entity entity1 = ENTITY1.builder().clearProperties().build(); + Entity entity2 = ENTITY2.builder() + .clearProperties() + .setProperty("bla", new NullValue()) + .build(); + Entity entity4 = new Entity.Builder(KEY2.newKey("newName1")) + .setProperty("value", new StringValue("value")) + .build(); + Entity entity5 = new Entity.Builder(KEY2.newKey("newName2")) + .setProperty("value", new StringValue("value")) + .build(); + batchWriter.add(entity4, entity5); + batchWriter.put(ENTITY3, entity1, entity2); + batchWriter.submit(); + Iterator entities = datastore.get(KEY1, KEY2, KEY3, entity4.key(), entity5.key()); + assertEquals(entity1, entities.next()); + assertEquals(entity2, entities.next()); + assertEquals(ENTITY3, entities.next()); + assertEquals(entity4, entities.next()); + assertEquals(entity5, entities.next()); + assertFalse(entities.hasNext()); - fail("Not yet implemented"); + try { + batchWriter.submit(); + } catch (DatastoreServiceException ex) { + // expected to fail + } + batchWriter = datastore.newBatchWriter(); + batchWriter.delete(entity4.key(), entity5.key()); + batchWriter.update(ENTITY1, ENTITY2, ENTITY3); + batchWriter.submit(); + entities = datastore.get(KEY1, KEY2, KEY3, entity4.key(), entity5.key()); + assertEquals(ENTITY1, entities.next()); + assertEquals(ENTITY2, entities.next()); + assertEquals(ENTITY3, entities.next()); + assertNull(entities.next()); + assertNull(entities.next()); + assertFalse(entities.hasNext()); + + // TODO need to cover more edge cases (ds failures, re-use of same entities,..) } @Test @@ -100,15 +144,15 @@ public void testAllocateId() { assertEquals(key1.kind(), pk1.kind()); assertTrue(key1.hasId()); assertFalse(key1.hasName()); - assertEquals(pk1.toKey(key1.id()), key1); + assertEquals(pk1.newKey(key1.id()), key1); Key key2 = datastore.allocateId(pk1); assertNotEquals(key1, key2); - assertEquals(pk1.toKey(key2.id()), key2); + assertEquals(pk1.newKey(key2.id()), key2); Key key3 = datastore.allocateId(key1); assertNotEquals(key1, key3); - assertEquals(pk1.toKey(key3.id()), key3); + assertEquals(pk1.newKey(key3.id()), key3); } @Test @@ -125,9 +169,9 @@ public void testAllocateIdArray() { map.put(++count, result.next()); } assertEquals(6, map.size()); - assertEquals(pKey1.toKey(map.get(1).id()), map.get(1)); - assertEquals(pKey1.toKey(map.get(5).id()), map.get(5)); - assertEquals(pKey2.toKey(map.get(2).id()), map.get(2)); + assertEquals(pKey1.newKey(map.get(1).id()), map.get(1)); + assertEquals(pKey1.newKey(map.get(5).id()), map.get(5)); + assertEquals(pKey2.newKey(map.get(2).id()), map.get(2)); assertEquals(key3.builder().id(map.get(3).id()).build(), map.get(3)); assertEquals(key3.builder().id(map.get(6).id()).build(), map.get(6)); assertEquals(key4.builder().id(map.get(4).id()).build(), map.get(4)); From 0baefcf6f8fc5488ec25f7d0ef33d2a8245feff6 Mon Sep 17 00:00:00 2001 From: aozarov Date: Fri, 5 Dec 2014 12:45:59 -0800 Subject: [PATCH 032/732] Added some tests for batchwrite --- .../google/gcloud/datastore/PartialKey.java | 4 +- .../datastore/DatastoreServiceTest.java | 62 ++++++++++++++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 8f7f0b085739..d51cd19cd621 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -242,14 +242,14 @@ public Builder builder() { return new Builder(dataset(), kind()).namespace(namespace()).addAncestors(ancestors()); } - public Key toKey(String name) { + public Key newKey(String name) { return new Key.Builder(dataset(), kind(), name) .namespace(namespace()) .addAncestors(ancestors()) .build(); } - public Key toKey(long id) { + public Key newKey(long id) { return new Key.Builder(dataset(), kind(), id) .namespace(namespace()) .addAncestors(ancestors()) diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 2c3914d2a4f6..4e9c23b421c1 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -26,7 +26,7 @@ public class DatastoreServiceTest { new BooleanValue.Builder(false).indexed(false).build(); private static final PartialKey PARTIAL_KEY1 = new PartialKey.Builder(DATASET, KIND1).build(); private static final PartialKey PARTIAL_KEY2 = new PartialKey.Builder(DATASET, KIND2).build(); - private static final Key KEY1 = PARTIAL_KEY1.toKey("name"); + private static final Key KEY1 = PARTIAL_KEY1.newKey("name"); private static final Key KEY2 = new Key.Builder(KEY1, KIND2, 1).build(); private static final Key KEY3 = KEY2.builder().name("bla").build(); private static final PartialEntity PARTIAL_ENTITY1 = new PartialEntity.Builder(PARTIAL_KEY2) @@ -74,14 +74,58 @@ public void testGetOptions() { } @Test - public void testNewTransaction() { + public void testNewTransactionCommit() { + fail("Not yet implemented"); + } + + @Test + public void testNewTransactionRollback() { fail("Not yet implemented"); } @Test public void testNewBatchWriter() { + BatchWriter batchWriter = datastore.newBatchWriter(); + Entity entity1 = ENTITY1.builder().clearProperties().build(); + Entity entity2 = ENTITY2.builder() + .clearProperties() + .setProperty("bla", new NullValue()) + .build(); + Entity entity4 = new Entity.Builder(KEY2.newKey("newName1")) + .setProperty("value", new StringValue("value")) + .build(); + Entity entity5 = new Entity.Builder(KEY2.newKey("newName2")) + .setProperty("value", new StringValue("value")) + .build(); + batchWriter.add(entity4, entity5); + batchWriter.put(ENTITY3, entity1, entity2); + batchWriter.submit(); + Iterator entities = datastore.get(KEY1, KEY2, KEY3, entity4.key(), entity5.key()); + assertEquals(entity1, entities.next()); + assertEquals(entity2, entities.next()); + assertEquals(ENTITY3, entities.next()); + assertEquals(entity4, entities.next()); + assertEquals(entity5, entities.next()); + assertFalse(entities.hasNext()); - fail("Not yet implemented"); + try { + batchWriter.submit(); + } catch (DatastoreServiceException ex) { + // expected to fail + } + batchWriter = datastore.newBatchWriter(); + batchWriter.delete(entity4.key(), entity5.key()); + batchWriter.update(ENTITY1, ENTITY2, ENTITY3); + batchWriter.submit(); + entities = datastore.get(KEY1, KEY2, KEY3, entity4.key(), entity5.key()); + assertEquals(ENTITY1, entities.next()); + assertEquals(ENTITY2, entities.next()); + assertEquals(ENTITY3, entities.next()); + assertNull(entities.next()); + assertNull(entities.next()); + assertFalse(entities.hasNext()); + + // TODO need to cover more edge cases (ds failures, re-use of same entities,..) } @Test @@ -100,15 +144,15 @@ public void testAllocateId() { assertEquals(key1.kind(), pk1.kind()); assertTrue(key1.hasId()); assertFalse(key1.hasName()); - assertEquals(pk1.toKey(key1.id()), key1); + assertEquals(pk1.newKey(key1.id()), key1); Key key2 = datastore.allocateId(pk1); assertNotEquals(key1, key2); - assertEquals(pk1.toKey(key2.id()), key2); + assertEquals(pk1.newKey(key2.id()), key2); Key key3 = datastore.allocateId(key1); assertNotEquals(key1, key3); - assertEquals(pk1.toKey(key3.id()), key3); + assertEquals(pk1.newKey(key3.id()), key3); } @Test @@ -125,9 +169,9 @@ public void testAllocateIdArray() { map.put(++count, result.next()); } assertEquals(6, map.size()); - assertEquals(pKey1.toKey(map.get(1).id()), map.get(1)); - assertEquals(pKey1.toKey(map.get(5).id()), map.get(5)); - assertEquals(pKey2.toKey(map.get(2).id()), map.get(2)); + assertEquals(pKey1.newKey(map.get(1).id()), map.get(1)); + assertEquals(pKey1.newKey(map.get(5).id()), map.get(5)); + assertEquals(pKey2.newKey(map.get(2).id()), map.get(2)); assertEquals(key3.builder().id(map.get(3).id()).build(), map.get(3)); assertEquals(key3.builder().id(map.get(6).id()).build(), map.get(6)); assertEquals(key4.builder().id(map.get(4).id()).build(), map.get(4)); From db7b76b2829ae2392dd3336109c9f55b0a1c7a04 Mon Sep 17 00:00:00 2001 From: aozarov Date: Sat, 6 Dec 2014 18:22:54 -0800 Subject: [PATCH 033/732] Adding PropertyContainer as a convinient way to add Properties --- .../gcloud/datastore/PropertyContainer.java | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 src/main/java/com/google/gcloud/datastore/PropertyContainer.java diff --git a/src/main/java/com/google/gcloud/datastore/PropertyContainer.java b/src/main/java/com/google/gcloud/datastore/PropertyContainer.java new file mode 100644 index 000000000000..a63026411d82 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/PropertyContainer.java @@ -0,0 +1,144 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.collect.ImmutableSortedMap; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * A container of properties (name and Value pairs). + */ +abstract class PropertyContainer + , B extends PropertyContainer.Builder> + extends Serializable { + + private static final long serialVersionUID = 8175618724683792766L; + + private final transient ImmutableSortedMap> properties; + + protected abstract static class Builder + , B extends Builder> { + + private final Map> properties; + + public Builder() { + properties = new HashMap<>(); + } + + public Builder(PropertyContainer entity) { + properties = new HashMap<>(entity.properties()); + } + + protected B self() { + return (B) this; + } + + public B clearProperties() { + properties.clear(); + return self(); + } + + public B removeProperty(String name) { + properties.remove(name); + return self(); + } + + public B setProperty(String name, Value value) { + properties.put(name, value); + return self(); + } + + setNullProperty(String name); + longValue + booleanValue + doubleValue + dateTimeValue + PartialEntityValue + KeyValue + ListValue + blobValue + rawValue // should work for all.... + + + + public E build() { + return build(ImmutableSortedMap.copyOf(properties)); + } + + protected abstract E build(ImmutableSortedMap> properties); + } + + protected PropertyContainer(ImmutableSortedMap> properties) { + this.properties = properties; + } + + public boolean hasProperty(String name) { + return properties.containsKey(name); + } + + @SuppressWarnings("unchecked") + public > V property(String name) { + return (V) properties.get(name); + } + + public boolean isNull(String name) { + return properties.get(name) instanceof NullValue; + } + + public String stringProperty(String name) { + return ((StringValue) property(name)).get(); + } + + longProperty + booleanValue + doubleValue + dateTimeValue + PartialEntityValue + KeyValue + ListValue + blobValue + rawValue // should work for all.... + + public Set propertyNames() { + return properties.keySet(); + } + + /** + * Returns a new builder for this entity (values are copied). + */ + public abstract B builder(); + + ImmutableSortedMap> properties() { + return properties; + } + + @Override + public int hashCode() { + return properties.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof PropertyContainer)) { + return false; + } + return properties.equals(((PropertyContainer) obj).properties); + } + + @Override + protected final DatastoreV1.Entity toPb() { + DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); + for (Map.Entry> entry : properties.entrySet()) { + DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); + propertyPb.setName(entry.getKey()); + propertyPb.setValue(entry.getValue().toPb()); + entityPb.addProperty(propertyPb.build()); + } + populateEntityBuilder(entityPb); + return entityPb.build(); + } + + protected abstract void populateEntityBuilder(DatastoreV1.Entity.Builder entity); +} From 31cf798de84c4214264ea1cb466e2590bf8c39e7 Mon Sep 17 00:00:00 2001 From: aozarov Date: Sun, 7 Dec 2014 13:09:17 -0800 Subject: [PATCH 034/732] add direct value set/get to Value --- .../gcloud/datastore/PropertyContainer.java | 145 +++++++++++++++--- .../datastore/DatastoreServiceTest.java | 1 + 2 files changed, 123 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/PropertyContainer.java b/src/main/java/com/google/gcloud/datastore/PropertyContainer.java index a63026411d82..8016561ef5a6 100644 --- a/src/main/java/com/google/gcloud/datastore/PropertyContainer.java +++ b/src/main/java/com/google/gcloud/datastore/PropertyContainer.java @@ -2,11 +2,18 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.common.collect.ImmutableSortedMap; +import com.google.gcloud.datastore.Value.Type; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; + +// TODO: make entity and Partial entity extends it and to tests + documentaion the option +// of list and direct value set/get. + /** * A container of properties (name and Value pairs). */ @@ -31,6 +38,7 @@ public Builder(PropertyContainer entity) { properties = new HashMap<>(entity.properties()); } + @SuppressWarnings("unchecked") protected B self() { return (B) this; } @@ -50,18 +58,60 @@ public B setProperty(String name, Value value) { return self(); } - setNullProperty(String name); - longValue - booleanValue - doubleValue - dateTimeValue - PartialEntityValue - KeyValue - ListValue - blobValue - rawValue // should work for all.... + public B setNullProperty(String name) { + properties.put(name, new NullValue()); + return self(); + } + + public B setStringProperty(String name, String value) { + properties.put(name, new StringValue(value)); + return self(); + } + + public B setLongProperty(String name, long value) { + properties.put(name, new LongValue(value)); + return self(); + } + + public B setDoubleProperty(String name, double value) { + properties.put(name, new DoubleValue(value)); + return self(); + } + + public B setBooleanProperty(String name, boolean value) { + properties.put(name, new BooleanValue(value)); + return self(); + } + + public B setDateAndTimeProperty(String name, DateAndTime value) { + properties.put(name, new DateAndTimeValue(value)); + return self(); + } + public B setKeyProperty(String name, Key value) { + properties.put(name, new KeyValue(value)); + return self(); + } + public B setPartialEntityProperty(String name, PartialEntity value) { + properties.put(name, new PartialEntityValue(value)); + return self(); + } + + public B setListProperty(String name, List> values) { + properties.put(name, new ListValue(values)); + return self(); + } + + public B setListProperty(String name, Value... value) { + properties.put(name, new ListValue(Arrays.asList(value))); + return self(); + } + + public B setBlobProperty(String name, Blob value) { + properties.put(name, new BlobValue(value)); + return self(); + } public E build() { return build(ImmutableSortedMap.copyOf(properties)); @@ -74,32 +124,81 @@ protected PropertyContainer(ImmutableSortedMap> propertie this.properties = properties; } + /** + * Returns {@code true} if there is such property with the given {@code name}. + */ public boolean hasProperty(String name) { return properties.containsKey(name); } - @SuppressWarnings("unchecked") + /** + * Returns the {@link Value} of property with the given {@code name}. + * + * @throws DatastoreServiceException if not such property. + */ public > V property(String name) { - return (V) properties.get(name); + @SuppressWarnings("unchecked") + V property = (V) properties.get(name); + if (property == null) { + throw DatastoreServiceException.throwInvalidRequest("No such property %s", name); + } + return property; } - public boolean isNull(String name) { - return properties.get(name) instanceof NullValue; + public Type propertyType(String name) { + return property(name).type(); + } + + public boolean isNullProperty(String name) { + return property(name) instanceof NullValue; } public String stringProperty(String name) { return ((StringValue) property(name)).get(); } - longProperty - booleanValue - doubleValue - dateTimeValue - PartialEntityValue - KeyValue - ListValue - blobValue - rawValue // should work for all.... + public long longProperty(String name) { + return ((LongValue) property(name)).get(); + } + + public double doubleProperty(String name) { + return ((DoubleValue) property(name)).get(); + } + + public boolean booleanProperty(String name) { + return ((BooleanValue) property(name)).get(); + } + + public DateAndTime dateAndTimeProperty(String name) { + return ((DateAndTimeValue) property(name)).get(); + } + + public Key keyProperty(String name) { + return ((KeyValue) property(name)).get(); + } + + public PartialEntity partialEntityProperty(String name) { + return ((PartialEntityValue) property(name)).get(); + } + + public List> listProperty(String name) { + return ((ListValue) property(name)).get(); + } + + public Blob blobProperty(String name) { + return ((BlobValue) property(name)).get(); + } + + /** + * Returns the property's value as {@code RawValue}. + */ + public RawValue rawValueProperty(String name) { + Value value = property(name); + if (value instanceof RawValue) { + return (RawValue) value; + } + return new RawValue(value.toPb()); + } public Set propertyNames() { return properties.keySet(); diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 4e9c23b421c1..1b5d9f9869d9 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -75,6 +75,7 @@ public void testGetOptions() { @Test public void testNewTransactionCommit() { + // TODO (also try list value with different types) fail("Not yet implemented"); } From 5a2398b0e7519a38375d8fefd4171498167a741d Mon Sep 17 00:00:00 2001 From: ozarov Date: Sun, 7 Dec 2014 22:59:53 -0800 Subject: [PATCH 035/732] work in progress --- .classpath | 10 ---------- .../com/google/gcloud/datastore/DateAndTime.java | 3 +-- .../com/google/gcloud/datastore/ListValue.java | 16 ++++++++-------- .../gcloud/datastore/DatastoreServiceTest.java | 9 +++++++++ 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.classpath b/.classpath index d6cf6121af66..9ed6ee4d0713 100644 --- a/.classpath +++ b/.classpath @@ -23,15 +23,5 @@ - - - - - - - - - - diff --git a/src/main/java/com/google/gcloud/datastore/DateAndTime.java b/src/main/java/com/google/gcloud/datastore/DateAndTime.java index 492802e3b8dc..38124f0526da 100644 --- a/src/main/java/com/google/gcloud/datastore/DateAndTime.java +++ b/src/main/java/com/google/gcloud/datastore/DateAndTime.java @@ -8,8 +8,7 @@ import java.util.Date; /** - * A Google Cloud Datastore timestamp. - * A Datastore timestamp is represented in micro-seconds. + * A Google Cloud Datastore timestamp (represented in micro-seconds). * This class is immutable. * * @see Google Cloud Datastore Entities, Properties, and Keys diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java index b8bdb7ec17d6..ef5c64a02e4f 100644 --- a/src/main/java/com/google/gcloud/datastore/ListValue.java +++ b/src/main/java/com/google/gcloud/datastore/ListValue.java @@ -10,12 +10,12 @@ import java.util.List; public final class ListValue extends - Value>, ListValue, ListValue.Builder> { + Value>, ListValue, ListValue.Builder> { private static final long serialVersionUID = -5461475706792576395L; - static final BaseMarshaller>, ListValue, Builder> MARSHALLER = - new BaseMarshaller>, ListValue, Builder>() { + static final BaseMarshaller>, ListValue, Builder> MARSHALLER = + new BaseMarshaller>, ListValue, Builder>() { @Override public int getProtoFieldId() { @@ -23,7 +23,7 @@ public int getProtoFieldId() { } @Override - public Builder newBuilder(List> values) { + public Builder newBuilder(List> values) { return new Builder().set(values); } @@ -45,7 +45,7 @@ protected void setValue(ListValue from, DatastoreV1.Value.Builder to) { }; public static final class Builder extends - Value.BaseBuilder>, ListValue, Builder> { + Value.BaseBuilder>, ListValue, Builder> { private ImmutableList.Builder> listBuilder = ImmutableList.builder(); @@ -74,7 +74,7 @@ public Builder addValue(Value first, Value... other) { * @see com.google.gcloud.datastore.Value.BaseBuilder#set(java.lang.Object) */ @Override - public Builder set(List> properties) { + public Builder set(List> properties) { listBuilder = ImmutableList.>builder(); for (Value property : properties) { addValue(property); @@ -83,7 +83,7 @@ public Builder set(List> properties) { } @Override - public List> get() { + public List> get() { return listBuilder.build(); } @@ -94,7 +94,7 @@ public ListValue build() { } } - public ListValue(List> properties) { + public ListValue(List> properties) { this(new Builder().set(properties)); } diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 1b5d9f9869d9..2ea5a57e9e2c 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -11,6 +11,7 @@ import org.junit.Before; import org.junit.Test; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -29,14 +30,22 @@ public class DatastoreServiceTest { private static final Key KEY1 = PARTIAL_KEY1.newKey("name"); private static final Key KEY2 = new Key.Builder(KEY1, KIND2, 1).build(); private static final Key KEY3 = KEY2.builder().name("bla").build(); + private static final KeyValue KEY_VALUE = new KeyValue(KEY1); + private static final ListValue LIST_VALUE1 = new ListValue.Builder() + .addValue(NULL_VALUE) + .addValue(STR_VALUE, BOOL_VALUE) + .build(); + private static final ListValue LIST_VALUE2 = new ListValue(Collections.singletonList(KEY_VALUE)); private static final PartialEntity PARTIAL_ENTITY1 = new PartialEntity.Builder(PARTIAL_KEY2) .setProperty("str", STR_VALUE) .setProperty("bool", BOOL_VALUE) + .setProperty("list", LIST_VALUE1) .build(); private static final Entity ENTITY1 = new Entity.Builder(KEY1) .setProperty("str", STR_VALUE) .setProperty("bool", BOOL_VALUE) .setProperty("partial1", new PartialEntityValue(PARTIAL_ENTITY1)) + .setProperty("list", LIST_VALUE2) .build(); private static final Entity ENTITY2 = new Entity.Builder(KEY2, ENTITY1) .removeProperty("str") From f6d541a3802de5ffe3aa4a56e30d3de8ed799476 Mon Sep 17 00:00:00 2001 From: ozarov Date: Sun, 7 Dec 2014 23:49:51 -0800 Subject: [PATCH 036/732] work in progress --- .../gcloud/datastore/DateAndTimeValue.java | 57 ------------- .../{DateAndTime.java => DateTime.java} | 18 ++--- .../gcloud/datastore/DateTimeValue.java | 57 +++++++++++++ .../com/google/gcloud/datastore/Entity.java | 41 ++++------ .../gcloud/datastore/PartialEntity.java | 80 +++++-------------- .../gcloud/datastore/PropertyContainer.java | 30 +++---- .../com/google/gcloud/datastore/Value.java | 4 +- .../google/gcloud/datastore/package-info.java | 15 ++-- .../datastore/DatastoreServiceTest.java | 12 +-- .../gcloud/datastore/SerializationTest.java | 6 +- 10 files changed, 127 insertions(+), 193 deletions(-) delete mode 100644 src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java rename src/main/java/com/google/gcloud/datastore/{DateAndTime.java => DateTime.java} (72%) create mode 100644 src/main/java/com/google/gcloud/datastore/DateTimeValue.java diff --git a/src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java deleted file mode 100644 index e1fa416c6bb1..000000000000 --- a/src/main/java/com/google/gcloud/datastore/DateAndTimeValue.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.google.gcloud.datastore; - -import static com.google.api.services.datastore.DatastoreV1.Value.TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER; - -import com.google.api.services.datastore.DatastoreV1; - -public final class DateAndTimeValue - extends Value { - - private static final long serialVersionUID = -5096238337676649540L; - - static final BaseMarshaller MARSHALLER = - new BaseMarshaller() { - - @Override - public int getProtoFieldId() { - return TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER; - } - - @Override - public Builder newBuilder(DateAndTime value) { - return new Builder(value); - } - - @Override - protected DateAndTime getValue(DatastoreV1.Value from) { - return new DateAndTime(from.getTimestampMicrosecondsValue()); - } - - @Override - protected void setValue(DateAndTimeValue from, DatastoreV1.Value.Builder to) { - to.setTimestampMicrosecondsValue(from.get().timestampMicroseconds()); - } - }; - - public static final class Builder - extends Value.BaseBuilder { - - public Builder(DateAndTime dateAndTime) { - super(Type.DATE_AND_TIME); - set(dateAndTime); - } - - @Override - public DateAndTimeValue build() { - return new DateAndTimeValue(this); - } - } - - public DateAndTimeValue(DateAndTime dateAndTime) { - this(new Builder(dateAndTime)); - } - - DateAndTimeValue(Builder builder) { - super(builder); - } -} diff --git a/src/main/java/com/google/gcloud/datastore/DateAndTime.java b/src/main/java/com/google/gcloud/datastore/DateTime.java similarity index 72% rename from src/main/java/com/google/gcloud/datastore/DateAndTime.java rename to src/main/java/com/google/gcloud/datastore/DateTime.java index 38124f0526da..7ade5c46d229 100644 --- a/src/main/java/com/google/gcloud/datastore/DateAndTime.java +++ b/src/main/java/com/google/gcloud/datastore/DateTime.java @@ -13,13 +13,13 @@ * * @see Google Cloud Datastore Entities, Properties, and Keys */ -public final class DateAndTime implements java.io.Serializable { +public final class DateTime implements java.io.Serializable { private static final long serialVersionUID = 7343324797621228378L; private final long timestampMicroseconds; - DateAndTime(long timestampMicroseconds) { + DateTime(long timestampMicroseconds) { this.timestampMicroseconds = timestampMicroseconds; } @@ -35,10 +35,10 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof DateAndTime)) { + if (!(obj instanceof DateTime)) { return false; } - return timestampMicroseconds == ((DateAndTime) obj).timestampMicroseconds; + return timestampMicroseconds == ((DateTime) obj).timestampMicroseconds; } public long timestampMicroseconds() { @@ -59,15 +59,15 @@ public Calendar toCalendar() { return cal; } - public static DateAndTime now() { - return new DateAndTime(System.nanoTime() / 1000L); + public static DateTime now() { + return new DateTime(System.nanoTime() / 1000L); } - public static DateAndTime copyFrom(Date date) { - return new DateAndTime(checkNotNull(date).getTime() * 1000L); + public static DateTime copyFrom(Date date) { + return new DateTime(checkNotNull(date).getTime() * 1000L); } - public static DateAndTime copyFrom(Calendar calendar) { + public static DateTime copyFrom(Calendar calendar) { return copyFrom(calendar.getTime()); } } diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java new file mode 100644 index 000000000000..fc2258c26c10 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java @@ -0,0 +1,57 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public final class DateTimeValue + extends Value { + + private static final long serialVersionUID = -5096238337676649540L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(DateTime value) { + return new Builder(value); + } + + @Override + protected DateTime getValue(DatastoreV1.Value from) { + return new DateTime(from.getTimestampMicrosecondsValue()); + } + + @Override + protected void setValue(DateTimeValue from, DatastoreV1.Value.Builder to) { + to.setTimestampMicrosecondsValue(from.get().timestampMicroseconds()); + } + }; + + public static final class Builder + extends Value.BaseBuilder { + + public Builder(DateTime dateTime) { + super(Type.DATE_TIME); + set(dateTime); + } + + @Override + public DateTimeValue build() { + return new DateTimeValue(this); + } + } + + public DateTimeValue(DateTime dateTime) { + this(new Builder(dateTime)); + } + + DateTimeValue(Builder builder) { + super(builder); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index 979174942e06..d6d671439d6f 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -1,5 +1,7 @@ package com.google.gcloud.datastore; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSortedMap; @@ -10,7 +12,7 @@ * An entity holds one or more properties, represented by a name (as {@link String}) * and a value (as {@link Value}), and is associated with a {@link Key}. * For a list of possible values see {@link Value.Type}. - * This class is immutable. To edit (a copy) use {@link #builder()}. + * This class is immutable. * * @see Google Cloud Datastore Entities, Properties, and Keys */ @@ -18,45 +20,35 @@ public final class Entity extends PartialEntity { private static final long serialVersionUID = 432961565733066915L; - public static final class Builder extends PartialEntity.Builder { + public static final class Builder extends PropertyContainer.Builder { + + private Key key; public Builder(Key key) { - super(key); + this.key = checkNotNull(key); } public Builder(Entity entity) { super(entity); + key = entity.key(); } /** * Create a Builder for the given key and with the properties from the given entity. */ public Builder(Key key, PartialEntity entity) { - super(key, entity); - } - - @Override - public Builder clearProperties() { - super.clearProperties(); - return this; - } - - @Override - public Builder removeProperty(String name) { - super.removeProperty(name); - return this; + super(entity); + this.key = key; } - @Override - public Builder setProperty(String name, Value value) { - super.setProperty(name, value); + public Builder key(Key key) { + this.key = checkNotNull(key); return this; } @Override - public Entity build() { - PartialEntity entity = super.build(); - return new Entity((Key) entity.key(), entity.properties()); + protected Entity build(ImmutableSortedMap> properties) { + return new Entity(key, properties); } } @@ -72,11 +64,6 @@ public Key key() { return (Key) super.key(); } - @Override - public Builder builder() { - return new Builder(this); - } - @Override protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { return fromPb(DatastoreV1.Entity.parseFrom(bytesPb)); diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index 7d22be430372..8e1e8578e291 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -4,71 +4,59 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.protobuf.InvalidProtocolBufferException; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; -import java.util.Set; /** * A partial entity holds one or more properties, represented by a name (as {@link String}) * and a value (as {@link Value}). * For a list of possible values see {@link Value.Type}. * A partial entity also can be associated with a key (partial or full). - * This class is immutable. To edit (a copy) use {@link #builder()}. + * This class is immutable. */ -public class PartialEntity extends Serializable { +public class PartialEntity extends PropertyContainer { private static final long serialVersionUID = 6492561268709192891L; private final transient PartialKey key; - private final transient ImmutableSortedMap> properties; - public static class Builder { + public static class Builder extends PropertyContainer.Builder { - private final PartialKey key; - private final Map> properties; + private PartialKey key; + + public Builder() { + } /** * Construct a builder with a partial key (could be null). */ public Builder(PartialKey key) { this.key = key; - properties = new HashMap<>(); } public Builder(PartialEntity entity) { + super(entity); key = entity.key(); - properties = new HashMap<>(entity.properties()); } public Builder(PartialKey key, PartialEntity entity) { + super(entity); this.key = key; - properties = new HashMap<>(entity.properties()); - } - - public Builder clearProperties() { - properties.clear(); - return this; } - public Builder removeProperty(String name) { - properties.remove(name); - return this; - } - - public Builder setProperty(String name, Value value) { - properties.put(name, value); + public Builder key(PartialKey key) { + this.key = key; return this; } - public PartialEntity build() { - return new PartialEntity(key, ImmutableSortedMap.copyOf(properties)); + @Override + protected PartialEntity build(ImmutableSortedMap> properties) { + return new PartialEntity(key, properties); } } protected PartialEntity(PartialKey key, ImmutableSortedMap> properties) { + super(properties); this.key = key; - this.properties = properties; } /** @@ -78,33 +66,9 @@ public PartialKey key() { return key; } - public boolean hasProperty(String name) { - return properties.containsKey(name); - } - - @SuppressWarnings("unchecked") - public > V property(String name) { - return (V) properties.get(name); - } - - public Set propertyNames() { - return properties.keySet(); - } - - /** - * Returns a new builder for this entity (values are copied). - */ - public Builder builder() { - return new Builder(this); - } - - ImmutableSortedMap> properties() { - return properties; - } - @Override public int hashCode() { - return Objects.hash(key, properties); + return Objects.hash(key, properties()); } @Override @@ -114,22 +78,14 @@ public boolean equals(Object obj) { } PartialEntity other = (PartialEntity) obj; return Objects.equals(key, other.key) - && Objects.equals(properties, other.properties); + && super.equals(obj); } @Override - protected DatastoreV1.Entity toPb() { - DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); + protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) { if (key != null) { entityPb.setKey(key.toPb()); } - for (Map.Entry> entry : properties.entrySet()) { - DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); - propertyPb.setName(entry.getKey()); - propertyPb.setValue(entry.getValue().toPb()); - entityPb.addProperty(propertyPb.build()); - } - return entityPb.build(); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/PropertyContainer.java b/src/main/java/com/google/gcloud/datastore/PropertyContainer.java index 8016561ef5a6..c2393ba2a741 100644 --- a/src/main/java/com/google/gcloud/datastore/PropertyContainer.java +++ b/src/main/java/com/google/gcloud/datastore/PropertyContainer.java @@ -10,23 +10,16 @@ import java.util.Map; import java.util.Set; - -// TODO: make entity and Partial entity extends it and to tests + documentaion the option -// of list and direct value set/get. - /** * A container of properties (name and Value pairs). */ -abstract class PropertyContainer - , B extends PropertyContainer.Builder> - extends Serializable { +abstract class PropertyContainer extends Serializable { private static final long serialVersionUID = 8175618724683792766L; private final transient ImmutableSortedMap> properties; - protected abstract static class Builder - , B extends Builder> { + protected abstract static class Builder> { private final Map> properties; @@ -34,7 +27,7 @@ public Builder() { properties = new HashMap<>(); } - public Builder(PropertyContainer entity) { + public Builder(PropertyContainer entity) { properties = new HashMap<>(entity.properties()); } @@ -83,8 +76,8 @@ public B setBooleanProperty(String name, boolean value) { return self(); } - public B setDateAndTimeProperty(String name, DateAndTime value) { - properties.put(name, new DateAndTimeValue(value)); + public B setDateAndTimeProperty(String name, DateTime value) { + properties.put(name, new DateTimeValue(value)); return self(); } @@ -169,8 +162,8 @@ public boolean booleanProperty(String name) { return ((BooleanValue) property(name)).get(); } - public DateAndTime dateAndTimeProperty(String name) { - return ((DateAndTimeValue) property(name)).get(); + public DateTime dateAndTimeProperty(String name) { + return ((DateTimeValue) property(name)).get(); } public Key keyProperty(String name) { @@ -204,11 +197,6 @@ public Set propertyNames() { return properties.keySet(); } - /** - * Returns a new builder for this entity (values are copied). - */ - public abstract B builder(); - ImmutableSortedMap> properties() { return properties; } @@ -223,7 +211,7 @@ public boolean equals(Object obj) { if (!(obj instanceof PropertyContainer)) { return false; } - return properties.equals(((PropertyContainer) obj).properties); + return properties.equals(((PropertyContainer) obj).properties); } @Override @@ -239,5 +227,5 @@ protected final DatastoreV1.Entity toPb() { return entityPb.build(); } - protected abstract void populateEntityBuilder(DatastoreV1.Entity.Builder entity); + protected abstract void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb); } diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java index 3d24423b65d8..03b40e61d98e 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -83,9 +83,9 @@ public enum Type { BOOLEAN(BooleanValue.MARSHALLER, BooleanValue.MARSHALLER), /** - * Represents a {@link DateAndTime} value. + * Represents a {@link DateTime} value. */ - DATE_AND_TIME(DateAndTimeValue.MARSHALLER, DateAndTimeValue.MARSHALLER), + DATE_TIME(DateTimeValue.MARSHALLER, DateTimeValue.MARSHALLER), /** * Represents a {@link Blob} value. diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java index a5ec1bad4c05..23b61cade70c 100644 --- a/src/main/java/com/google/gcloud/datastore/package-info.java +++ b/src/main/java/com/google/gcloud/datastore/package-info.java @@ -9,18 +9,21 @@ * Entity entity = datastore.get(key); * if (entity == null) { * entity = new Entity.Builder(key) - * .setProperty("name", new StringValue("John Do")) + * .setStringProperty("name", "John Do") * .setProperty("age", new LongValue.Builder(100).indexed(false).build()) - * .setProperty("updated", new BooleanValue(false)) + * .setBooleanProperty("updated", false) * .build(); * datastore.put(entity); * } else { - * BooleanValue updated = entity.property("updated"); - * if (!updated.get()) { + * boolean updated = entity.booleanProperty("updated"); + * if (!updated.get) { + * String[] name = entity.stringProperty("name").split(" "); * entity = entity.builder() - * .setProperty("updated", new BooleanValue(true)) + * .setStringProperty("name", name[0]) + * .setProperty("last_name", new StringProperty.Builder(name[1]).indexed(false).build()) + * .setBooleanProperty("updated", true) * .removeProperty("old_property") - * .setProperty("new_property", new DoubleValue(1.1)) + * .setDoubleProperty("new_property", 1.1) * .build(); * datastore.update(entity); * } diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 2ea5a57e9e2c..3cc103450e0c 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -96,16 +96,16 @@ public void testNewTransactionRollback() { @Test public void testNewBatchWriter() { BatchWriter batchWriter = datastore.newBatchWriter(); - Entity entity1 = ENTITY1.builder().clearProperties().build(); - Entity entity2 = ENTITY2.builder() + Entity entity1 = new Entity.Builder(ENTITY1).clearProperties().build(); + Entity entity2 = new Entity.Builder(ENTITY2) .clearProperties() - .setProperty("bla", new NullValue()) + .setNullProperty("bla") .build(); Entity entity4 = new Entity.Builder(KEY2.newKey("newName1")) .setProperty("value", new StringValue("value")) .build(); Entity entity5 = new Entity.Builder(KEY2.newKey("newName2")) - .setProperty("value", new StringValue("value")) + .setStringProperty("value", "value") .build(); batchWriter.add(entity4, entity5); batchWriter.put(ENTITY3, entity1, entity2); @@ -244,7 +244,7 @@ public void testUpdate() { } datastore.add(ENTITY3); assertEquals(ENTITY3, datastore.get(ENTITY3.key())); - Entity entity3 = ENTITY3.builder() + Entity entity3 = new Entity.Builder(ENTITY3) .clearProperties() .setProperty("bla", new NullValue()) .build(); @@ -261,7 +261,7 @@ public void testPut() { assertNull(keys.next()); assertFalse(keys.hasNext()); - Entity entity2 = ENTITY2.builder() + Entity entity2 = new Entity.Builder(ENTITY2) .clearProperties() .setProperty("bla", new NullValue()) .build(); diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index d450d84f08b8..58a2b14d4d1c 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -30,8 +30,8 @@ public class SerializationTest { private static final LongValue LONG_VALUE = new LongValue(123); private static final DoubleValue DOUBLE_VALUE = new DoubleValue(12.34); private static final BooleanValue BOOLEAN_VALUE = new BooleanValue(true); - private static final DateAndTimeValue DATE_AND_TIME_VALUE = - new DateAndTimeValue(DateAndTime.now()); + private static final DateTimeValue DATE_AND_TIME_VALUE = + new DateTimeValue(DateTime.now()); private static final BlobValue BLOB_VALUE = new BlobValue(Blob.copyFrom(new byte[] {10, 0, -2})); private static final RawValue RAW_VALUE = new RawValue( @@ -76,7 +76,7 @@ public class SerializationTest { .put(Type.LONG, LONG_VALUE) .put(Type.DOUBLE, DOUBLE_VALUE) .put(Type.BOOLEAN, BOOLEAN_VALUE) - .put(Type.DATE_AND_TIME, DATE_AND_TIME_VALUE) + .put(Type.DATE_TIME, DATE_AND_TIME_VALUE) .put(Type.BLOB, BLOB_VALUE) .put(Type.RAW_VALUE, RAW_VALUE) .build(); From e9f8f4918cfa7ff3ab7800a9fee47c9397d697c0 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 8 Dec 2014 09:00:31 -0800 Subject: [PATCH 037/732] remove Key's builder --- .../java/com/google/gcloud/datastore/Key.java | 18 ++++++++++-------- .../google/gcloud/datastore/PartialKey.java | 13 ++++++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index c490dded05f5..14c27114a589 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -14,7 +14,7 @@ /** * A key that is guaranteed to be complete and could be used to reference a * Google Cloud Datastore {@link Entity}. - * This class is immutable. To edit (a copy) use {@link #builder()}. + * This class is immutable. * * @see Google Cloud Datastore Entities, Properties, and Keys */ @@ -47,6 +47,15 @@ public Builder(Key parent, String kind, long id) { this.id = id; } + public Builder(Key from) { + super(from); + if (from.hasId()) { + id = from.id(); + } else { + name = from.name(); + } + } + @Override public Builder addAncestor(String kind, long id) { super.addAncestor(kind, id); @@ -122,13 +131,6 @@ private Key(PartialKey key, long id) { super(key.dataset(), key.namespace(), newPath(key, id)); } - @Override - public Builder builder() { - Builder builder = - hasId() ? new Builder(dataset(), kind(), id()) : new Builder(dataset(), kind(), name()); - return builder.namespace(namespace()).addAncestors(ancestors()); - } - public boolean hasId() { return id() != null; } diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index d51cd19cd621..51ece279e54c 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -18,7 +18,7 @@ /** * A partial key (without a name or id). * Could be used as metadata for {@link PartialEntity}. - * This class is immutable. To edit (a copy) use {@link #builder()}. + * This class is immutable. */ public class PartialKey extends Serializable { @@ -142,6 +142,13 @@ public Builder(Key parent, String kind) { this.kind = kind; } + public Builder(PartialKey from) { + dataset = from.dataset(); + namespace = from.namespace(); + kind = from.kind(); + ancestors.addAll(from.ancestors()); + } + public Builder addAncestor(String kind, long id) { checkArgument(id != 0, "id must not be equal to zero"); return addAncestor(new Ancestor(kind, id)); @@ -238,10 +245,6 @@ public String kind() { return getLeaf().kind(); } - public Builder builder() { - return new Builder(dataset(), kind()).namespace(namespace()).addAncestors(ancestors()); - } - public Key newKey(String name) { return new Key.Builder(dataset(), kind(), name) .namespace(namespace()) From d67e7e35139c7ca5a67d7b8e0d4e3a4defa7deb1 Mon Sep 17 00:00:00 2001 From: aozarov Date: Mon, 8 Dec 2014 17:39:06 -0800 Subject: [PATCH 038/732] Change Builders to factory style --- .../com/google/gcloud/ServiceOptions.java | 2 +- ...PropertyContainer.java => BaseEntity.java} | 18 +- .../com/google/gcloud/datastore/BaseKey.java | 193 +++++++++++ .../google/gcloud/datastore/BlobValue.java | 13 +- .../google/gcloud/datastore/BooleanValue.java | 13 +- .../datastore/DatastoreServiceImpl.java | 9 +- .../datastore/DatastoreServiceOptions.java | 13 +- .../gcloud/datastore/DateTimeValue.java | 13 +- .../google/gcloud/datastore/DoubleValue.java | 13 +- .../com/google/gcloud/datastore/Entity.java | 33 +- .../java/com/google/gcloud/datastore/Key.java | 177 ++++------ .../google/gcloud/datastore/KeyBuilder.java | 52 +-- .../gcloud/datastore/KeyPathElement.java | 101 ++++++ .../com/google/gcloud/datastore/KeyValue.java | 13 +- .../google/gcloud/datastore/ListValue.java | 13 +- .../google/gcloud/datastore/LongValue.java | 13 +- .../google/gcloud/datastore/NullValue.java | 12 +- .../gcloud/datastore/PartialEntity.java | 51 +-- .../google/gcloud/datastore/PartialKey.java | 315 +++--------------- .../com/google/gcloud/datastore/RawValue.java | 27 +- .../google/gcloud/datastore/StringValue.java | 13 +- .../com/google/gcloud/datastore/Value.java | 4 +- .../google/gcloud/datastore/package-info.java | 9 +- .../datastore/DatastoreServiceTest.java | 39 +-- .../gcloud/datastore/SerializationTest.java | 46 ++- 25 files changed, 622 insertions(+), 583 deletions(-) rename src/main/java/com/google/gcloud/datastore/{PropertyContainer.java => BaseEntity.java} (90%) create mode 100644 src/main/java/com/google/gcloud/datastore/BaseKey.java create mode 100644 src/main/java/com/google/gcloud/datastore/KeyPathElement.java diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java index 908710ceb750..826f59e1d300 100644 --- a/src/main/java/com/google/gcloud/ServiceOptions.java +++ b/src/main/java/com/google/gcloud/ServiceOptions.java @@ -70,7 +70,7 @@ protected abstract static class Builder> { private HttpTransport httpTransport; private AuthConfig authConfig; - public Builder() {} + protected Builder() {} protected Builder(ServiceOptions options) { host = options.host; diff --git a/src/main/java/com/google/gcloud/datastore/PropertyContainer.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java similarity index 90% rename from src/main/java/com/google/gcloud/datastore/PropertyContainer.java rename to src/main/java/com/google/gcloud/datastore/BaseEntity.java index c2393ba2a741..7aae77d360e2 100644 --- a/src/main/java/com/google/gcloud/datastore/PropertyContainer.java +++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java @@ -11,23 +11,23 @@ import java.util.Set; /** - * A container of properties (name and Value pairs). + * A base class for entities. */ -abstract class PropertyContainer extends Serializable { +abstract class BaseEntity extends Serializable { private static final long serialVersionUID = 8175618724683792766L; private final transient ImmutableSortedMap> properties; - protected abstract static class Builder> { + protected abstract static class Builder> { private final Map> properties; - public Builder() { + protected Builder() { properties = new HashMap<>(); } - public Builder(PropertyContainer entity) { + protected Builder(BaseEntity entity) { properties = new HashMap<>(entity.properties()); } @@ -91,7 +91,7 @@ public B setPartialEntityProperty(String name, PartialEntity value) { return self(); } - public B setListProperty(String name, List> values) { + public B setListProperty(String name, List> values) { properties.put(name, new ListValue(values)); return self(); } @@ -113,7 +113,7 @@ public E build() { protected abstract E build(ImmutableSortedMap> properties); } - protected PropertyContainer(ImmutableSortedMap> properties) { + protected BaseEntity(ImmutableSortedMap> properties) { this.properties = properties; } @@ -208,10 +208,10 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof PropertyContainer)) { + if (!(obj instanceof BaseEntity)) { return false; } - return properties.equals(((PropertyContainer) obj).properties); + return properties.equals(((BaseEntity) obj).properties); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java new file mode 100644 index 000000000000..47859cd619d4 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java @@ -0,0 +1,193 @@ +package com.google.gcloud.datastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; +import static com.google.gcloud.datastore.DatastoreServiceOptions.validateNamespace; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; + +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; + +/** + * Base class for keys. + */ +abstract class BaseKey extends Serializable { + + private static final long serialVersionUID = -4671243265877410635L; + + private final transient String dataset; + private final transient String namespace; + private final transient ImmutableList ancestors; + private final transient String kind; + + abstract static class Builder> { + + private String dataset; + private String namespace; + private String kind; + private final List ancestors; + + private static final int MAX_PATH = 100; + + public Builder(String dataset, String kind) { + this.dataset = validateDataset(dataset); + this.kind = validateKind(kind); + ancestors = new LinkedList<>(); + } + + public Builder(BaseKey copyFrom) { + dataset = copyFrom.dataset(); + namespace = copyFrom.namespace(); + kind = copyFrom.kind(); + ancestors = new LinkedList<>(copyFrom.ancestors()); + } + + @SuppressWarnings("unchecked") + protected B self() { + return (B) this; + } + + public B addAncestor(String kind, long id) { + checkArgument(id != 0, "id must not be equal to zero"); + return addAncestor(new KeyPathElement(kind, id)); + } + + public B addAncestor(String kind, String name) { + checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); + checkArgument(name.length() <= 500, "name must not exceed 500 characters"); + return addAncestor(new KeyPathElement(kind, name)); + } + + public B addAncestor(KeyPathElement... ancestor) { + Preconditions.checkState(ancestors.size() + ancestor.length <= MAX_PATH, + "path can have at most 100 elements"); + for (KeyPathElement pathElement : ancestor) { + ancestors.add(pathElement); + } + return self(); + } + + public B addAncestors(Iterable ancestors) { + for (KeyPathElement pathElement : ancestors) { + addAncestor(pathElement); + } + return self(); + } + + public B kind(String kind) { + this.kind = validateKind(kind); + return self(); + } + + private String validateKind(String kind) { + checkArgument(!Strings.isNullOrEmpty(kind), "kind must not be empty or null"); + checkArgument(kind.length() <= 500, "kind must not contain more than 500 characters"); + return kind; + } + + public B clearPath() { + ancestors.clear(); + return self(); + } + + public B dataset(String dataset) { + this.dataset = validateDataset(dataset); + return self(); + } + + public B namespace(String namespace) { + this.namespace = validateNamespace(namespace); + return self(); + } + + public K build() { + return build(dataset, namespace, ImmutableList.copyOf(ancestors), kind); + } + + protected abstract K build( + String dataset, String namespace, ImmutableList ancestors, String kind); + } + + BaseKey(String dataset, String namespace, ImmutableList ancestors, String kind) { + this.dataset = dataset; + this.namespace = namespace; + this.ancestors = ancestors; + this.kind = kind; + } + + /** + * Returns the key's dataset. + */ + public String dataset() { + return dataset; + } + + /** + * Returns the key's namespace or {@code null} if not provided. + */ + public String namespace() { + return namespace; + } + + /** + * Returns an immutable list with the key's ancestors. + */ + public List ancestors() { + return ancestors; + } + + /** + * Returns the key's kind. + */ + public String kind() { + return kind; + } + + @Override + public int hashCode() { + return Objects.hash(dataset, namespace, ancestors, kind); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof BaseKey)) { + return false; + } + BaseKey other = (BaseKey) obj; + return Objects.equals(dataset, other.dataset) + && Objects.equals(namespace, other.namespace) + && Objects.equals(ancestors, other.ancestors) + && Objects.equals(kind, other.kind); + } + + @Override + protected DatastoreV1.Key toPb() { + DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); + DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder(); + if (dataset != null) { + partitionIdPb.setDatasetId(dataset); + } + if (namespace != null) { + partitionIdPb.setNamespace(namespace); + } + if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { + keyPb.setPartitionId(partitionIdPb.build()); + } + for (KeyPathElement pathEntry : ancestors) { + keyPb.addPathElement(pathEntry.toPb()); + } + addLeaf(keyPb); + return keyPb.build(); + } + + protected void addLeaf(DatastoreV1.Key.Builder keyPb) { + if (kind != null) { + keyPb.addPathElement(new KeyPathElement(kind).toPb()); + } + } +} diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java index d44d384909a6..0b3a3fed9129 100644 --- a/src/main/java/com/google/gcloud/datastore/BlobValue.java +++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java @@ -18,7 +18,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(Blob value) { - return new Builder(value); + return builder(value); } @Override @@ -34,9 +34,8 @@ protected void setValue(BlobValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(Blob blob) { + private Builder() { super(Type.BLOB); - set(blob); } @Override @@ -46,10 +45,14 @@ public BlobValue build() { } public BlobValue(Blob blob) { - this(new Builder(blob)); + this(builder(blob)); } - BlobValue(Builder builder) { + private BlobValue(Builder builder) { super(builder); } + + public static Builder builder(Blob blob) { + return new Builder().set(blob); + } } diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/src/main/java/com/google/gcloud/datastore/BooleanValue.java index 6eebdbf69b56..533ee14d5b0d 100644 --- a/src/main/java/com/google/gcloud/datastore/BooleanValue.java +++ b/src/main/java/com/google/gcloud/datastore/BooleanValue.java @@ -18,7 +18,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(Boolean value) { - return new Builder(value); + return builder(value); } @Override @@ -34,9 +34,8 @@ protected void setValue(BooleanValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(boolean value) { + private Builder() { super(Type.BOOLEAN); - set(value); } @Override @@ -46,10 +45,14 @@ public BooleanValue build() { } public BooleanValue(boolean value) { - this(new Builder(value)); + this(builder(value)); } - BooleanValue(Builder builder) { + private BooleanValue(Builder builder) { super(builder); } + + public static Builder builder(boolean value) { + return new Builder().set(value); + } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 69819b1c73f0..250fe0048360 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -87,13 +87,10 @@ protected Key computeNext() { } private PartialKey trimNameOrId(PartialKey key) { - if (key.getLeaf().nameOrId() == null) { - return key; + if (key instanceof Key) { + return PartialKey.builder(key).build(); } - return new PartialKey.Builder(key.dataset(), key.kind()) - .namespace(key.namespace()) - .addAncestors(key.ancestors()) - .build(); + return key; } @Override diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java index 8b2f906283d9..2f41df15a855 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java @@ -41,9 +41,10 @@ public static class Builder extends ServiceOptions.Builder { private String namespace; private boolean force = false; - public Builder() {} + private Builder() { + } - public Builder(DatastoreServiceOptions options) { + private Builder(DatastoreServiceOptions options) { super(options); dataset = options.dataset; force = options.force; @@ -125,4 +126,12 @@ DatastoreOptions toDatastoreOptions() { .initializer(httpRequestInitializer()) .build(); } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(DatastoreServiceOptions options) { + return new Builder(options); + } } diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java index fc2258c26c10..85ac6a853779 100644 --- a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java +++ b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java @@ -19,7 +19,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(DateTime value) { - return new Builder(value); + return builder(value); } @Override @@ -36,9 +36,8 @@ protected void setValue(DateTimeValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(DateTime dateTime) { + private Builder() { super(Type.DATE_TIME); - set(dateTime); } @Override @@ -48,10 +47,14 @@ public DateTimeValue build() { } public DateTimeValue(DateTime dateTime) { - this(new Builder(dateTime)); + this(builder(dateTime)); } - DateTimeValue(Builder builder) { + private DateTimeValue(Builder builder) { super(builder); } + + public static Builder builder(DateTime dateTime) { + return new Builder().set(dateTime); + } } diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/src/main/java/com/google/gcloud/datastore/DoubleValue.java index dca443a32851..b2f4e264c723 100644 --- a/src/main/java/com/google/gcloud/datastore/DoubleValue.java +++ b/src/main/java/com/google/gcloud/datastore/DoubleValue.java @@ -18,7 +18,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(Double value) { - return new Builder(value); + return builder(value); } @Override @@ -34,9 +34,8 @@ protected void setValue(DoubleValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(double value) { + public Builder() { super(Type.DOUBLE); - set(value); } @Override @@ -46,10 +45,14 @@ public DoubleValue build() { } public DoubleValue(double value) { - this(new Builder(value)); + this(builder(value)); } - DoubleValue(Builder builder) { + private DoubleValue(Builder builder) { super(builder); } + + public static Builder builder(double value) { + return new Builder().set(value); + } } diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index d6d671439d6f..f497cb6c7d67 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -20,27 +20,19 @@ public final class Entity extends PartialEntity { private static final long serialVersionUID = 432961565733066915L; - public static final class Builder extends PropertyContainer.Builder { + public static final class Builder extends BaseEntity.Builder { private Key key; - public Builder(Key key) { + private Builder(Key key) { this.key = checkNotNull(key); } - public Builder(Entity entity) { + private Builder(Entity entity) { super(entity); key = entity.key(); } - /** - * Create a Builder for the given key and with the properties from the given entity. - */ - public Builder(Key key, PartialEntity entity) { - super(entity); - this.key = key; - } - public Builder key(Key key) { this.key = checkNotNull(key); return this; @@ -52,7 +44,7 @@ protected Entity build(ImmutableSortedMap> properties) { } } - private Entity(Key key, ImmutableSortedMap> properties) { + Entity(Key key, ImmutableSortedMap> properties) { super(key, properties); } @@ -70,11 +62,16 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { } static Entity fromPb(DatastoreV1.Entity entityPb) { - Preconditions.checkArgument(entityPb.hasKey()); - Builder builder = new Builder(Key.fromPb(entityPb.getKey())); - for (DatastoreV1.Property property : entityPb.getPropertyList()) { - builder.setProperty(property.getName(), Value.fromPb(property.getValue())); - } - return builder.build(); + PartialEntity entity = PartialEntity.fromPb(entityPb); + Preconditions.checkState(entity instanceof Entity, "Entity is not complete"); + return (Entity) entity; + } + + public static Builder builder(Key key) { + return new Builder(key); + } + + public static Builder builder(Entity copyFrom) { + return new Builder(copyFrom); } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index 14c27114a589..c73f5d8819dc 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -3,6 +3,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; @@ -22,88 +23,33 @@ public final class Key extends PartialKey { private static final long serialVersionUID = 3160994559785491356L; - public static final class Builder extends PartialKey.Builder { + private final transient String name; + private final transient Long id; + + public static final class Builder extends BaseKey.Builder { private String name; private Long id; - public Builder(String dataset, String kind, String name) { + private Builder(String dataset, String kind, String name) { super(dataset, kind); this.name = name; } - public Builder(String dataset, String kind, long id) { + private Builder(String dataset, String kind, long id) { super(dataset, kind); this.id = id; } - public Builder(Key parent, String kind, String name) { - super(parent, kind); - this.name = name; - } - - public Builder(Key parent, String kind, long id) { - super(parent, kind); - this.id = id; - } - - public Builder(Key from) { - super(from); - if (from.hasId()) { - id = from.id(); + private Builder(Key copyFrom) { + super(copyFrom); + if (copyFrom.hasId()) { + id = copyFrom.id(); } else { - name = from.name(); + name = copyFrom.name(); } } - @Override - public Builder addAncestor(String kind, long id) { - super.addAncestor(kind, id); - return this; - } - - @Override - public Builder addAncestor(String kind, String name) { - super.addAncestor(kind, name); - return this; - } - - @Override - public Builder addAncestor(Ancestor... ancestor) { - super.addAncestor(ancestor); - return this; - } - - @Override - public Builder addAncestors(Iterable ancestors) { - super.addAncestors(ancestors); - return this; - } - - @Override - public Builder kind(String kind) { - super.kind(kind); - return this; - } - - @Override - public Builder clearPath() { - super.clearPath(); - return this; - } - - @Override - public Builder dataset(String dataset) { - super.dataset(dataset); - return this; - } - - @Override - public Builder namespace(String namespace) { - super.namespace(namespace); - return this; - } - public Builder name(String name) { this.name = name; id = null; @@ -117,48 +63,56 @@ public Builder id(long id) { } @Override - public Key build() { - PartialKey key = super.build(); - return id == null ? new Key(key, name) : new Key(key, id); + protected Key build(String dataset, String namespace, ImmutableList ancestors, + String kind) { + if (id == null) { + return new Key(dataset, namespace, ancestors, kind, name); + } + return new Key(dataset, namespace, ancestors, kind, id); } } - private Key(PartialKey key, String name) { - super(key.dataset(), key.namespace(), newPath(key, name)); + Key(String dataset, String namespace, ImmutableList ancestors, + String kind, String name) { + super(dataset, namespace, ancestors, kind); + this.name = name; + this.id = null; } - private Key(PartialKey key, long id) { - super(key.dataset(), key.namespace(), newPath(key, id)); + Key(String dataset, String namespace, ImmutableList ancestors, + String kind, long id) { + super(dataset, namespace, ancestors, kind); + this.id = id; + this.name = null; } public boolean hasId() { - return id() != null; + return id != null; } /** * Returns the key's id or {@code null} if it has a name instead. */ public Long id() { - return getLeaf().id(); + return id; } public boolean hasName() { - return name() != null; + return name != null; } /** * Returns the key's name or {@code null} if it has an id instead. */ public String name() { - return getLeaf().name(); + return name; } /** * Returns the key's id (as {@link #Long}) or name (as {@link String}). */ public Object nameOrId() { - Ancestor leaf = getLeaf(); - return leaf.hasId() ? leaf.id() : leaf.name(); + return hasId() ? id : name; } /** @@ -181,7 +135,7 @@ public static Key fromUrlSafe(String urlSafe) { try { String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name()); DatastoreV1.Key keyPb = DatastoreV1.Key.parseFrom(ByteString.copyFromUtf8(utf8Str)); - return Key.fromPb(keyPb); + return fromPb(keyPb); } catch (UnsupportedEncodingException e) { throw new RuntimeException("Unxpeced decoding exception", e); } catch (InvalidProtocolBufferException e) { @@ -189,45 +143,48 @@ public static Key fromUrlSafe(String urlSafe) { } } - /** - * Convert an {@code IncompleteKey} to a {@code Key} provided that the key has - * either name or id (complete). - - * @throws IllegalArgumentException if provided key is not complete. - */ - public static Key fromIncompleteKey(PartialKey key) { - if (key instanceof Key) { - return (Key) key; - } - Ancestor leaf = key.getLeaf(); - if (leaf.hasId()) { - return new Key(key, leaf.id()); - } else if (leaf.hasName()) { - return new Key(key, leaf.name()); - } - throw new IllegalArgumentException("Key is missing name or id"); - } - @Override protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { return fromPb(DatastoreV1.Key.parseFrom(bytesPb)); } static Key fromPb(DatastoreV1.Key keyPb) { - return fromIncompleteKey(PartialKey.fromPb(keyPb)); + PartialKey key = PartialKey.fromPb(keyPb); + Preconditions.checkState(key instanceof Key, "Key is not complete"); + return (Key) key; + } + + public static Builder builder(String dataset, String kind, String name) { + return new Builder(dataset, kind, name); } - private static ImmutableList newPath(PartialKey key, String name) { - ImmutableList.Builder path = ImmutableList.builder(); - path.addAll(key.ancestors()); - path.add(new Ancestor(key.kind(), name)); - return path.build(); + public static Builder builder(String dataset, String kind, long id) { + return new Builder(dataset, kind, id); } - private static ImmutableList newPath(PartialKey key, long id) { - ImmutableList.Builder path = ImmutableList.builder(); - path.addAll(key.ancestors()); - path.add(new Ancestor(key.kind(), id)); - return path.build(); + public static Builder builder(Key copyFrom) { + return new Builder(copyFrom); + } + + public static Builder builder(Key parent, String kind, String name) { + Builder builder = builder(parent.dataset(), kind, name); + addParentToBuilder(parent, builder); + return builder; + } + + public static Builder builder(Key parent, String kind, long id) { + Builder builder = builder(parent.dataset(), kind, id); + addParentToBuilder(parent, builder); + return builder; + } + + private static void addParentToBuilder(Key parent, Builder builder) { + builder.namespace(parent.namespace()); + builder.addAncestors(parent.ancestors()); + if (parent.hasId()) { + builder.addAncestor(new KeyPathElement(parent.kind(), parent.id())); + } else { + builder.addAncestor(new KeyPathElement(parent.kind(), parent.name())); + } } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java index 8b9954717858..855c09d3358f 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -2,13 +2,13 @@ import static com.google.common.base.Preconditions.checkNotNull; -import com.google.gcloud.datastore.PartialKey.Ancestor; +import com.google.common.collect.ImmutableList; /** * An helper for creating keys for a specific {@link DatastoreService}, * using its associated dataset and namespace. */ -public final class KeyBuilder extends PartialKey.Builder { +public final class KeyBuilder extends BaseKey.Builder { private final DatastoreService service; @@ -22,55 +22,17 @@ public KeyBuilder(DatastoreService service, String kind) { } @Override - public KeyBuilder kind(String kind) { - super.kind(kind); - return this; - } - - @Override - public KeyBuilder addAncestor(String kind, String name) { - super.addAncestor(kind, name); - return this; - } - - @Override - public KeyBuilder addAncestor(String kind, long id) { - super.addAncestor(kind, id); - return this; - } - - @Override - public KeyBuilder namespace(String namespace) { - super.namespace(namespace); - return this; - } - - @Override - public KeyBuilder addAncestor(Ancestor... ancestor) { - super.addAncestor(ancestor); - return this; - } - - @Override - public KeyBuilder addAncestors(Iterable ancestors) { - super.addAncestors(ancestors); - return this; + protected PartialKey build(String dataset, String namespace, + ImmutableList ancestors, String kind) { + return new PartialKey(dataset, namespace, ancestors, kind); } public Key build(String name) { - PartialKey key = build(); - return new Key.Builder(key.dataset(), key.kind(), name) - .namespace(key.namespace()) - .addAncestors(key.ancestors()) - .build(); + return build().newKey(name); } public Key build(long id) { - PartialKey key = build(); - return new Key.Builder(key.dataset(), key.kind(), id) - .namespace(key.namespace()) - .addAncestors(key.ancestors()) - .build(); + return build().newKey(id); } /** diff --git a/src/main/java/com/google/gcloud/datastore/KeyPathElement.java b/src/main/java/com/google/gcloud/datastore/KeyPathElement.java new file mode 100644 index 000000000000..78b529ed6e7b --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/KeyPathElement.java @@ -0,0 +1,101 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.protobuf.InvalidProtocolBufferException; + +import java.util.Objects; + +/** + * Represents a single element in a key's path. + */ +public final class KeyPathElement extends Serializable { + + private static final long serialVersionUID = -7968078857690784595L; + + private final transient String kind; + private final transient Long id; + private final transient String name; + + KeyPathElement(String kind) { + this(kind, null); + } + + public KeyPathElement(String kind, long id) { + this.kind = kind; + this.id = id; + name = null; + } + + public KeyPathElement(String kind, String name) { + this.kind = kind; + this.name = name; + id = null; + } + + public String kind() { + return kind; + } + + public boolean hasId() { + return id != null; + } + + public Long id() { + return id; + } + + public boolean hasName() { + return name != null; + } + + public String name() { + return name; + } + + public Object nameOrId() { + return id == null ? name : id; + } + + @Override + public int hashCode() { + return Objects.hash(kind, id, name); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof KeyPathElement)) { + return false; + } + KeyPathElement other = (KeyPathElement) obj; + return Objects.equals(kind, other.kind) + && Objects.equals(id, other.id) + && Objects.equals(name, other.name); + } + + @Override + protected DatastoreV1.Key.PathElement toPb() { + DatastoreV1.Key.PathElement.Builder pathElementPb = DatastoreV1.Key.PathElement.newBuilder(); + pathElementPb.setKind(kind); + if (id != null) { + pathElementPb.setId(id); + } else if (name != null) { + pathElementPb.setName(name); + } + return pathElementPb.build(); + } + + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Key.PathElement.parseFrom(bytesPb)); + } + + static KeyPathElement fromPb(DatastoreV1.Key.PathElement pathElementPb) { + String kind = pathElementPb.getKind(); + if (pathElementPb.hasId()) { + return new KeyPathElement(kind, pathElementPb.getId()); + } else if (pathElementPb.hasName()) { + return new KeyPathElement(kind, pathElementPb.getName()); + } + return new KeyPathElement(kind); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java index 671e9c355b14..8a6db4ff74a4 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyValue.java +++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java @@ -18,7 +18,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(Key key) { - return new Builder(key); + return builder(key); } @Override @@ -34,9 +34,8 @@ protected void setValue(KeyValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(Key key) { + public Builder() { super(Type.KEY); - set(key); } @Override @@ -46,10 +45,14 @@ public KeyValue build() { } public KeyValue(Key key) { - this(new Builder(key)); + this(builder(key)); } - KeyValue(Builder builder) { + private KeyValue(Builder builder) { super(builder); } + + public static Builder builder(Key key) { + return new Builder().set(key); + } } diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java index ef5c64a02e4f..ee829a71dcc0 100644 --- a/src/main/java/com/google/gcloud/datastore/ListValue.java +++ b/src/main/java/com/google/gcloud/datastore/ListValue.java @@ -24,7 +24,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(List> values) { - return new Builder().set(values); + return builder().set(values); } @Override @@ -49,9 +49,8 @@ public static final class Builder extends private ImmutableList.Builder> listBuilder = ImmutableList.builder(); - public Builder() { + private Builder() { super(Type.LIST); - indexed(false); } public Builder addValue(Value value) { @@ -95,14 +94,18 @@ public ListValue build() { } public ListValue(List> properties) { - this(new Builder().set(properties)); + this(builder().set(properties)); } public ListValue(Value first, Value... other) { this(new Builder().addValue(first, other)); } - ListValue(Builder builder) { + private ListValue(Builder builder) { super(builder); } + + public static Builder builder() { + return new Builder().indexed(false); + } } diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/src/main/java/com/google/gcloud/datastore/LongValue.java index 9a4690a24133..97b789e344bd 100644 --- a/src/main/java/com/google/gcloud/datastore/LongValue.java +++ b/src/main/java/com/google/gcloud/datastore/LongValue.java @@ -18,7 +18,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(Long value) { - return new Builder(value); + return builder(value); } @Override @@ -34,9 +34,8 @@ protected void setValue(LongValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(long value) { + private Builder() { super(Type.LONG); - set(value); } @Override @@ -46,10 +45,14 @@ public LongValue build() { } public LongValue(long value) { - this(new Builder(value)); + this(builder(value)); } - LongValue(Builder builder) { + private LongValue(Builder builder) { super(builder); } + + public static Builder builder(long value) { + return new Builder().set(value); + } } diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java index fb60ab20042f..bb2c084522ae 100644 --- a/src/main/java/com/google/gcloud/datastore/NullValue.java +++ b/src/main/java/com/google/gcloud/datastore/NullValue.java @@ -13,7 +13,7 @@ public final class NullValue extends Value { @Override public Builder newBuilder(Void value) { - return new Builder(); + return builder(); } @Override @@ -34,7 +34,7 @@ protected void setValue(NullValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder() { + private Builder() { super(Type.NULL); } @@ -51,10 +51,14 @@ public Builder set(Void value) { } public NullValue() { - this(new Builder()); + this(builder()); } - NullValue(Builder builder) { + private NullValue(Builder builder) { super(builder); } + + public static Builder builder() { + return new Builder(); + } } diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index 8e1e8578e291..0796b2c9f5a1 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -13,36 +13,24 @@ * A partial entity also can be associated with a key (partial or full). * This class is immutable. */ -public class PartialEntity extends PropertyContainer { +public class PartialEntity extends BaseEntity { private static final long serialVersionUID = 6492561268709192891L; private final transient PartialKey key; - public static class Builder extends PropertyContainer.Builder { + public static class Builder extends BaseEntity.Builder { private PartialKey key; - public Builder() { + private Builder() { } - /** - * Construct a builder with a partial key (could be null). - */ - public Builder(PartialKey key) { - this.key = key; - } - - public Builder(PartialEntity entity) { + private Builder(PartialEntity entity) { super(entity); key = entity.key(); } - public Builder(PartialKey key, PartialEntity entity) { - super(entity); - this.key = key; - } - public Builder key(PartialKey key) { this.key = key; return this; @@ -59,6 +47,10 @@ protected PartialEntity(PartialKey key, ImmutableSortedMap>copyOf(properties())); + } + /** * Returns the key for this entity or {@code null} if it does not have one. */ @@ -94,11 +86,30 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { } static PartialEntity fromPb(DatastoreV1.Entity entityPb) { - PartialKey key = entityPb.hasKey() ? PartialKey.fromPb(entityPb.getKey()) : null; - Builder builder = new Builder(key); + ImmutableSortedMap.Builder> properties = + ImmutableSortedMap.naturalOrder(); for (DatastoreV1.Property property : entityPb.getPropertyList()) { - builder.setProperty(property.getName(), Value.fromPb(property.getValue())); + properties.put(property.getName(), Value.fromPb(property.getValue())); + } + PartialKey partialKey = null; + if (entityPb.hasKey()) { + partialKey = PartialKey.fromPb(entityPb.getKey()); + if (partialKey instanceof Key) { + return new Entity((Key) partialKey, properties.build()); + } } - return builder.build(); + return new PartialEntity(partialKey, properties.build()); + } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(PartialKey key) { + return new Builder().key(key); + } + + public static Builder builder(PartialEntity copyFrom) { + return new Builder(copyFrom); } } diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 51ece279e54c..129dbdf6e87d 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -1,267 +1,49 @@ package com.google.gcloud.datastore; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset; -import static com.google.gcloud.datastore.DatastoreServiceOptions.validateNamespace; - import com.google.api.services.datastore.DatastoreV1; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import com.google.api.services.datastore.DatastoreV1.Key.PathElement; import com.google.common.collect.ImmutableList; import com.google.protobuf.InvalidProtocolBufferException; -import java.util.LinkedList; import java.util.List; -import java.util.Objects; /** * A partial key (without a name or id). * Could be used as metadata for {@link PartialEntity}. * This class is immutable. */ -public class PartialKey extends Serializable { +public class PartialKey extends BaseKey { private static final long serialVersionUID = -75301206578793347L; - private final transient String dataset; - private final transient String namespace; - private final transient ImmutableList ancestors; - - public static final class Ancestor extends Serializable { - - private static final long serialVersionUID = -7968078857690784595L; - - private final transient String kind; - private final transient Long id; - private final transient String name; - - private Ancestor(String kind) { - this(kind, null); - } - - public Ancestor(String kind, long id) { - this.kind = kind; - this.id = id; - name = null; - } - - public Ancestor(String kind, String name) { - this.kind = kind; - this.name = name; - id = null; - } - - public String kind() { - return kind; - } - - public boolean hasId() { - return id != null; - } - - public Long id() { - return id; - } - - public boolean hasName() { - return name != null; - } - - public String name() { - return name; - } - - public Object nameOrId() { - return id == null ? name : id; - } - - @Override - public int hashCode() { - return Objects.hash(kind, id, name); - } + public static class Builder extends BaseKey.Builder { - @Override - public boolean equals(Object obj) { - if (!(obj instanceof Ancestor)) { - return false; - } - Ancestor other = (Ancestor) obj; - return Objects.equals(kind, other.kind) - && Objects.equals(id, other.id) - && Objects.equals(name, other.name); + private Builder(String dataset, String kind) { + super(dataset, kind); } - @Override - protected DatastoreV1.Key.PathElement toPb() { - DatastoreV1.Key.PathElement.Builder pathElementPb = DatastoreV1.Key.PathElement.newBuilder(); - pathElementPb.setKind(kind); - if (id != null) { - pathElementPb.setId(id); - } else if (name != null) { - pathElementPb.setName(name); - } - return pathElementPb.build(); + private Builder(PartialKey copyFrom) { + super(copyFrom); } @Override - protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { - return fromPb(DatastoreV1.Key.PathElement.parseFrom(bytesPb)); - } - - static Ancestor fromPb(DatastoreV1.Key.PathElement pathElementPb) { - String kind = pathElementPb.getKind(); - if (pathElementPb.hasId()) { - return new Ancestor(kind, pathElementPb.getId()); - } else if (pathElementPb.hasName()) { - return new Ancestor(kind, pathElementPb.getName()); - } - return new Ancestor(kind); + protected PartialKey build(String dataset, String namespace, + ImmutableList ancestors, String kind) { + return new PartialKey(dataset, namespace, ancestors, kind); } } - public static class Builder { - - private String dataset; - private String namespace; - private String kind; - private final List ancestors = new LinkedList<>(); - - private static final int MAX_PATH = 100; - - public Builder(String dataset, String kind) { - this.dataset = validateDataset(dataset); - this.kind = validateKind(kind); - } - - public Builder(Key parent, String kind) { - dataset = parent.dataset(); - namespace = parent.namespace(); - ancestors.addAll(parent.ancestors()); - ancestors.add(parent.getLeaf()); - this.kind = kind; - } - - public Builder(PartialKey from) { - dataset = from.dataset(); - namespace = from.namespace(); - kind = from.kind(); - ancestors.addAll(from.ancestors()); - } - - public Builder addAncestor(String kind, long id) { - checkArgument(id != 0, "id must not be equal to zero"); - return addAncestor(new Ancestor(kind, id)); - } - - public Builder addAncestor(String kind, String name) { - checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); - checkArgument(name.length() <= 500, "name must not exceed 500 characters"); - return addAncestor(new Ancestor(kind, name)); - } - - public Builder addAncestor(Ancestor... ancestor) { - Preconditions.checkState(ancestors.size() + ancestor.length <= MAX_PATH, - "path can have at most 100 elements"); - for (Ancestor pathElement : ancestor) { - ancestors.add(pathElement); - } - return this; - } - - public Builder addAncestors(Iterable ancestors) { - for (Ancestor pathElement : ancestors) { - addAncestor(pathElement); - } - return this; - } - - public Builder kind(String kind) { - this.kind = validateKind(kind); - return this; - } - - private String validateKind(String kind) { - checkArgument(!Strings.isNullOrEmpty(kind), "kind must not be empty or null"); - checkArgument(kind.length() <= 500, "kind must not contain more than 500 characters"); - return kind; - } - - public Builder clearPath() { - ancestors.clear(); - return this; - } - - public Builder dataset(String dataset) { - this.dataset = validateDataset(dataset); - return this; - } - - public Builder namespace(String namespace) { - this.namespace = validateNamespace(namespace); - return this; - } - - public PartialKey build() { - Ancestor leaf = new Ancestor(kind); - ImmutableList pathList = - ImmutableList.builder().addAll(ancestors).add(leaf).build(); - return new PartialKey(dataset, namespace, pathList); - } - } - - PartialKey(String dataset, String namespace, ImmutableList path) { - checkState(!path.isEmpty(), "path must not be empty"); - this.dataset = dataset; - this.namespace = namespace; - this.ancestors = path; - } - - /** - * Returns the key's dataset. - */ - public String dataset() { - return dataset; - } - - /** - * Returns the key's namespace or {@code null} if not provided. - */ - public String namespace() { - return namespace; - } - - /** - * Returns an immutable list with the key's ancestors. - */ - public List ancestors() { - return ancestors.subList(0, ancestors.size() - 1); - } - - /** - * Returns the key's kind. - */ - public String kind() { - return getLeaf().kind(); + PartialKey(String dataset, String namespace, ImmutableList ancestors, + String kind) { + super(dataset, namespace, ancestors, kind); } public Key newKey(String name) { - return new Key.Builder(dataset(), kind(), name) - .namespace(namespace()) - .addAncestors(ancestors()) - .build(); + return new Key(dataset(), namespace(), ImmutableList.copyOf(ancestors()), kind(), name); } public Key newKey(long id) { - return new Key.Builder(dataset(), kind(), id) - .namespace(namespace()) - .addAncestors(ancestors()) - .build(); - } - - @Override - public int hashCode() { - return Objects.hash(dataset, namespace, ancestors); + return new Key(dataset(), namespace(), ImmutableList.copyOf(ancestors()), kind(), id); } @Override @@ -269,33 +51,7 @@ public boolean equals(Object obj) { if (!(obj instanceof PartialKey)) { return false; } - PartialKey other = (PartialKey) obj; - return Objects.equals(dataset, other.dataset) - && Objects.equals(namespace, other.namespace) - && Objects.equals(ancestors, other.ancestors); - } - - @Override - protected DatastoreV1.Key toPb() { - return toPb(this).build(); - } - - static DatastoreV1.Key.Builder toPb(PartialKey key) { - DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); - DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder(); - if (key.dataset != null) { - partitionIdPb.setDatasetId(key.dataset); - } - if (key.namespace != null) { - partitionIdPb.setNamespace(key.namespace); - } - if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { - keyPb.setPartitionId(partitionIdPb.build()); - } - for (Ancestor pathEntry : key.ancestors) { - keyPb.addPathElement(pathEntry.toPb()); - } - return keyPb; + return super.equals(obj); } @Override @@ -315,14 +71,41 @@ static PartialKey fromPb(DatastoreV1.Key keyPb) { namespace = partitionIdPb.getNamespace(); } } - ImmutableList.Builder pathBuilder = ImmutableList.builder(); - for (DatastoreV1.Key.PathElement pathElementPb : keyPb.getPathElementList()) { - pathBuilder.add(Ancestor.fromPb(pathElementPb)); + List pathElementsPb = keyPb.getPathElementList(); + if (pathElementsPb.isEmpty()) { + return new PartialKey(dataset, namespace, ImmutableList.of(), null); + } + ImmutableList.Builder pathBuilder = ImmutableList.builder(); + for (int i = 0; i < pathElementsPb.size() - 1; i++) { + pathBuilder.add(KeyPathElement.fromPb(pathElementsPb.get(i))); + } + PathElement leaf = pathElementsPb.get(pathElementsPb.size() - 1); + String kind = leaf.getKind(); + if (leaf.hasId()) { + return new Key(dataset, namespace, pathBuilder.build(), kind, leaf.getId()); + } else if (leaf.hasName()) { + return new Key(dataset, namespace, pathBuilder.build(), kind, leaf.getName()); } - return new PartialKey(dataset, namespace, pathBuilder.build()); + return new PartialKey(dataset, namespace, pathBuilder.build(), kind); } - Ancestor getLeaf() { - return ancestors.get(ancestors.size() - 1); + public static Builder builder(String dataset, String kind) { + return new Builder(dataset, kind); + } + + public static Builder builder(PartialKey copyFrom) { + return new Builder(copyFrom); + } + + public static Builder builder(Key parent, String kind) { + Builder builder = new Builder(parent.dataset(), kind) + .namespace(parent.namespace()) + .addAncestors(parent.ancestors()); + if (parent.hasId()) { + builder.addAncestor(new KeyPathElement(parent.kind(), parent.id())); + } else { + builder.addAncestor(new KeyPathElement(parent.kind(), parent.name())); + } + return builder; } } diff --git a/src/main/java/com/google/gcloud/datastore/RawValue.java b/src/main/java/com/google/gcloud/datastore/RawValue.java index 120ad7085e4e..aa4c2229e96c 100644 --- a/src/main/java/com/google/gcloud/datastore/RawValue.java +++ b/src/main/java/com/google/gcloud/datastore/RawValue.java @@ -11,7 +11,7 @@ public final class RawValue extends Value { - Builder(DatastoreV1.Value valuePb) { + private Builder() { super(Type.RAW_VALUE); - if (valuePb.hasIndexed()) { - indexed(valuePb.getIndexed()); - } - if (valuePb.hasMeaning()) { - meaning(valuePb.getMeaning()); - } - set(valuePb); } @Override @@ -49,11 +42,23 @@ public RawValue build() { } } - RawValue(Builder builder) { + private RawValue(Builder builder) { super(builder); } RawValue(DatastoreV1.Value valuePb) { - this(new Builder(valuePb)); + this(builder(valuePb)); + } + + static Builder builder(DatastoreV1.Value valuePb) { + Builder builder = new Builder(); + if (valuePb.hasIndexed()) { + builder.indexed(valuePb.getIndexed()); + } + if (valuePb.hasMeaning()) { + builder.meaning(valuePb.getMeaning()); + } + builder.set(valuePb); + return builder; } } diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java index 7f8ee6e979d8..4d35aa3e3e01 100644 --- a/src/main/java/com/google/gcloud/datastore/StringValue.java +++ b/src/main/java/com/google/gcloud/datastore/StringValue.java @@ -19,7 +19,7 @@ public int getProtoFieldId() { @Override public Builder newBuilder(String value) { - return new Builder(value); + return builder(value); } @Override @@ -35,9 +35,8 @@ protected void setValue(StringValue from, DatastoreV1.Value.Builder to) { public static final class Builder extends Value.BaseBuilder { - public Builder(String value) { + private Builder() { super(Type.STRING); - set(value); } @Override @@ -50,10 +49,14 @@ public StringValue build() { } public StringValue(String value) { - this(new Builder(value)); + this(builder(value)); } - StringValue(Builder builder) { + private StringValue(Builder builder) { super(builder); } + + public static Builder builder(String value) { + return new Builder().set(value); + } } diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java index 03b40e61d98e..d35820e38db7 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -16,7 +16,7 @@ /** * Base class for all Google Cloud Datastore value types. * All values must be associated with a non-null content (except {@link NullValue}). - * All values are immutable (including their content). To edit (a copy) use {@link #builder()}. + * All values are immutable (including their content). To edit (a copy) use {@link #toBuilder()}. * Unsupported value (deprecated or unrecognized) would be represented by {@link RawValue}. * * @param the type of the content for this value @@ -298,7 +298,7 @@ public final V get() { } @SuppressWarnings("unchecked") - public final B builder() { + public final B toBuilder() { BuilderFactory builderFactory = type().getBuilderFactory(); B builder = builderFactory.newBuilder(get()); return builder.mergeFrom((P) this); diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java index 23b61cade70c..60aaddb13a74 100644 --- a/src/main/java/com/google/gcloud/datastore/package-info.java +++ b/src/main/java/com/google/gcloud/datastore/package-info.java @@ -2,15 +2,15 @@ * A client to the Google Cloud Datastore. * Typical usage would be: * 
 {@code
- * DatastoreServiceOptions options = new DatastoreServiceOptions.Builder().dataset(DATASET).build();
+ * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
  * KeyBuilder keyBuilder = datastore.newKeyBuilder(kind);
  * Key key = keyBuilder.build(keyName);
  * Entity entity = datastore.get(key);
  * if (entity == null) {
- *   entity = new Entity.Builder(key)
+ *   entity = Entity.builder(key)
  *       .setStringProperty("name", "John Do")
- *       .setProperty("age", new LongValue.Builder(100).indexed(false).build())
+ *       .setProperty("age", LongValue.builder(100).indexed(false).build())
  *       .setBooleanProperty("updated", false)
  *       .build();
  *   datastore.put(entity);
@@ -20,7 +20,7 @@
  *     String[] name = entity.stringProperty("name").split(" ");
  *     entity = entity.builder()
  *         .setStringProperty("name", name[0])
- *         .setProperty("last_name", new StringProperty.Builder(name[1]).indexed(false).build())
+ *         .setProperty("last_name", StringProperty.builder(name[1]).indexed(false).build())
  *         .setBooleanProperty("updated", true)
  *         .removeProperty("old_property")
  *         .setDoubleProperty("new_property", 1.1)
@@ -29,6 +29,5 @@
  *   }
  * }
  * }
- * */ package com.google.gcloud.datastore; diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 3cc103450e0c..970da50d56d5 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -23,35 +23,36 @@ public class DatastoreServiceTest { private static final String KIND2 = "kind2"; private static final NullValue NULL_VALUE = new NullValue(); private static final StringValue STR_VALUE = new StringValue("str"); - private static final BooleanValue BOOL_VALUE = - new BooleanValue.Builder(false).indexed(false).build(); - private static final PartialKey PARTIAL_KEY1 = new PartialKey.Builder(DATASET, KIND1).build(); - private static final PartialKey PARTIAL_KEY2 = new PartialKey.Builder(DATASET, KIND2).build(); + private static final BooleanValue BOOL_VALUE = BooleanValue.builder(false).indexed(false).build(); + private static final PartialKey PARTIAL_KEY1 = PartialKey.builder(DATASET, KIND1).build(); + private static final PartialKey PARTIAL_KEY2 = PartialKey.builder(DATASET, KIND2).build(); private static final Key KEY1 = PARTIAL_KEY1.newKey("name"); - private static final Key KEY2 = new Key.Builder(KEY1, KIND2, 1).build(); - private static final Key KEY3 = KEY2.builder().name("bla").build(); + private static final Key KEY2 = Key.builder(KEY1, KIND2, 1).build(); + private static final Key KEY3 = Key.builder(KEY2).name("bla").build(); private static final KeyValue KEY_VALUE = new KeyValue(KEY1); - private static final ListValue LIST_VALUE1 = new ListValue.Builder() + private static final ListValue LIST_VALUE1 = ListValue.builder() .addValue(NULL_VALUE) .addValue(STR_VALUE, BOOL_VALUE) .build(); private static final ListValue LIST_VALUE2 = new ListValue(Collections.singletonList(KEY_VALUE)); - private static final PartialEntity PARTIAL_ENTITY1 = new PartialEntity.Builder(PARTIAL_KEY2) + private static final PartialEntity PARTIAL_ENTITY1 = PartialEntity.builder(PARTIAL_KEY2) .setProperty("str", STR_VALUE) .setProperty("bool", BOOL_VALUE) .setProperty("list", LIST_VALUE1) .build(); - private static final Entity ENTITY1 = new Entity.Builder(KEY1) + private static final Entity ENTITY1 = Entity.builder(KEY1) .setProperty("str", STR_VALUE) .setProperty("bool", BOOL_VALUE) .setProperty("partial1", new PartialEntityValue(PARTIAL_ENTITY1)) .setProperty("list", LIST_VALUE2) .build(); - private static final Entity ENTITY2 = new Entity.Builder(KEY2, ENTITY1) + private static final Entity ENTITY2 = Entity.builder(ENTITY1) + .key(KEY2) .removeProperty("str") .setProperty("null", NULL_VALUE) .build(); - private static final Entity ENTITY3 = new Entity.Builder(KEY3, ENTITY1) + private static final Entity ENTITY3 = Entity.builder(ENTITY1) + .key(KEY3) .removeProperty("str") .setProperty("null", NULL_VALUE) .setProperty("partial2", new PartialEntityValue(ENTITY1)) @@ -67,7 +68,7 @@ public void setUp() { // reference: https://cloud.google.com/datastore/docs/tools/devserver // Or even better, using a "GCE_HOME" param/env initiate and destroy the server // before and after tests via ant or maven - options = new DatastoreServiceOptions.Builder() + options = DatastoreServiceOptions.builder() .dataset(DATASET) .host("http://localhost:8080") .build(); @@ -96,15 +97,15 @@ public void testNewTransactionRollback() { @Test public void testNewBatchWriter() { BatchWriter batchWriter = datastore.newBatchWriter(); - Entity entity1 = new Entity.Builder(ENTITY1).clearProperties().build(); - Entity entity2 = new Entity.Builder(ENTITY2) + Entity entity1 = Entity.builder(ENTITY1).clearProperties().build(); + Entity entity2 = Entity.builder(ENTITY2) .clearProperties() .setNullProperty("bla") .build(); - Entity entity4 = new Entity.Builder(KEY2.newKey("newName1")) + Entity entity4 = Entity.builder(KEY2.newKey("newName1")) .setProperty("value", new StringValue("value")) .build(); - Entity entity5 = new Entity.Builder(KEY2.newKey("newName2")) + Entity entity5 = Entity.builder(KEY2.newKey("newName2")) .setStringProperty("value", "value") .build(); batchWriter.add(entity4, entity5); @@ -244,7 +245,7 @@ public void testUpdate() { } datastore.add(ENTITY3); assertEquals(ENTITY3, datastore.get(ENTITY3.key())); - Entity entity3 = new Entity.Builder(ENTITY3) + Entity entity3 = Entity.builder(ENTITY3) .clearProperties() .setProperty("bla", new NullValue()) .build(); @@ -261,7 +262,7 @@ public void testPut() { assertNull(keys.next()); assertFalse(keys.hasNext()); - Entity entity2 = new Entity.Builder(ENTITY2) + Entity entity2 = Entity.builder(ENTITY2) .clearProperties() .setProperty("bla", new NullValue()) .build(); @@ -293,7 +294,7 @@ public void testDelete() { public void testNewKeyBuilder() { KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1); assertEquals(PARTIAL_KEY1, keyBuilder.build()); - assertEquals(PARTIAL_KEY1.builder().kind(KIND2).build(), + assertEquals(PartialKey.builder(PARTIAL_KEY1).kind(KIND2).build(), datastore.newKeyBuilder(KIND2).build()); assertEquals(KEY1, keyBuilder.build("name")); assertEquals(KEY1.builder().id(2).build(), keyBuilder.build(2)); diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 58a2b14d4d1c..a7b7c87baf7f 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -18,14 +18,13 @@ public class SerializationTest { private static final PartialKey INCOMPLETE_KEY1 = - new PartialKey.Builder("ds", "k").addAncestor("p", 1).build(); - private static final Key KEY1 = new Key.Builder("ds", "k", "n").build(); + PartialKey.builder("ds", "k").addAncestor("p", 1).build(); + private static final Key KEY1 = Key.builder("ds", "k", "n").build(); private static final PartialKey INCOMPLETE_KEY2 = - new PartialKey.Builder(KEY1, "v").addAncestor("p", 1).build(); - private static final Key KEY2 = new Key.Builder(KEY1, "v", 2).build(); + PartialKey.builder(KEY1, "v").addAncestor("p", 1).build(); + private static final Key KEY2 = Key.builder(KEY1, "v", 2).build(); private static final KeyValue KEY_VALUE = new KeyValue(KEY1); - private static final NullValue NULL_VALUE = - new NullValue.Builder().indexed(true).build(); + private static final NullValue NULL_VALUE = NullValue.builder().indexed(true).build(); private static final StringValue STRING_VALUE = new StringValue("hello"); private static final LongValue LONG_VALUE = new LongValue(123); private static final DoubleValue DOUBLE_VALUE = new DoubleValue(12.34); @@ -36,34 +35,31 @@ public class SerializationTest { new BlobValue(Blob.copyFrom(new byte[] {10, 0, -2})); private static final RawValue RAW_VALUE = new RawValue( DatastoreV1.Value.newBuilder().setBlobKeyValue("blob-key").setMeaning(18).build()); - private static final Entity ENTITY1 = new Entity.Builder(KEY1).build(); + private static final Entity ENTITY1 = Entity.builder(KEY1).build(); private static final Entity ENTITY2 = - new Entity.Builder(KEY2).setProperty("null", new NullValue()).build(); - private static final Entity ENTITY3 = - new Entity.Builder(KEY2) - .setProperty("p1", new StringValue.Builder("hi1").meaning(10).build()) - .setProperty("p2", new StringValue.Builder("hi2").meaning(11).indexed(false).build()) - .setProperty("p3", new LongValue.Builder(100).indexed(false).meaning(100).build()) - .build(); + Entity.builder(KEY2).setProperty("null", new NullValue()).build(); + private static final Entity ENTITY3 = Entity.builder(KEY2) + .setProperty("p1", StringValue.builder("hi1").meaning(10).build()) + .setProperty("p2", StringValue.builder("hi2").meaning(11).indexed(false).build()) + .setProperty("p3", LongValue.builder(100).indexed(false).meaning(100).build()) + .build(); private static final PartialEntity EMBEDDED_ENTITY1 = ENTITY1; private static final PartialEntity EMBEDDED_ENTITY2 = ENTITY2; - private static final PartialEntity EMBEDDED_ENTITY3 = - new PartialEntity.Builder(INCOMPLETE_KEY1) - .setProperty("p1", STRING_VALUE) - .setProperty("p2", new LongValue.Builder(100).indexed(false).meaning(100).build()) - .build(); + private static final PartialEntity EMBEDDED_ENTITY3 = PartialEntity.builder(INCOMPLETE_KEY1) + .setProperty("p1", STRING_VALUE) + .setProperty("p2", LongValue.builder(100).indexed(false).meaning(100).build()) + .build(); private static final PartialEntityValue EMBEDDED_ENTITY_VALUE1 = new PartialEntityValue(EMBEDDED_ENTITY1); private static final PartialEntityValue EMBEDDED_ENTITY_VALUE2 = new PartialEntityValue(EMBEDDED_ENTITY2); private static final PartialEntityValue EMBEDDED_ENTITY_VALUE3 = new PartialEntityValue(EMBEDDED_ENTITY3); - private static final ListValue LIST_VALUE = - new ListValue.Builder() - .addValue(NULL_VALUE) - .addValue(STRING_VALUE) - .addValue(new NullValue()) - .build(); + private static final ListValue LIST_VALUE = ListValue.builder() + .addValue(NULL_VALUE) + .addValue(STRING_VALUE) + .addValue(new NullValue()) + .build(); @SuppressWarnings("rawtypes") private Multimap typeToValues = ImmutableMultimap.builder() From aab8eefb30684863bbc89bd0789985c23490b9c6 Mon Sep 17 00:00:00 2001 From: ozarov Date: Mon, 8 Dec 2014 22:24:08 -0800 Subject: [PATCH 039/732] Fix tests after builder refactoring. --- .../java/com/google/gcloud/datastore/Key.java | 28 ++++++++- .../google/gcloud/datastore/ListValue.java | 8 ++- .../datastore/DatastoreServiceTest.java | 62 +++++++++++++------ .../gcloud/datastore/SerializationTest.java | 1 + 4 files changed, 78 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index c73f5d8819dc..f569ed811194 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -11,6 +11,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; +import java.util.Objects; /** * A key that is guaranteed to be complete and could be used to reference a @@ -76,14 +77,14 @@ protected Key build(String dataset, String namespace, ImmutableList ancestors, String kind, long id) { super(dataset, namespace, ancestors, kind); this.id = id; - this.name = null; + name = null; } public boolean hasId() { @@ -143,6 +144,29 @@ public static Key fromUrlSafe(String urlSafe) { } } + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), name, id); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Key)) { + return false; + } + Key other = (Key) obj; + return Objects.equals(name, other.name) + && Objects.equals(id, other.id) + && super.equals(obj); + } + + @Override + protected void addLeaf(DatastoreV1.Key.Builder keyPb) { + KeyPathElement leaf = + hasId() ? new KeyPathElement(kind(), id) : new KeyPathElement(kind(), name); + keyPb.addPathElement(leaf.toPb()); + } + @Override protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { return fromPb(DatastoreV1.Key.parseFrom(bytesPb)); diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java index ee829a71dcc0..883b46f0fc28 100644 --- a/src/main/java/com/google/gcloud/datastore/ListValue.java +++ b/src/main/java/com/google/gcloud/datastore/ListValue.java @@ -67,6 +67,12 @@ public Builder addValue(Value first, Value... other) { return this; } + @Override + public Builder indexed(boolean indexed) { + DatastoreServiceException.throwInvalidRequest("ListValue can't specify index"); + return this; + } + /** * Copy the list of values. * @@ -106,6 +112,6 @@ private ListValue(Builder builder) { } public static Builder builder() { - return new Builder().indexed(false); + return new Builder(); } } diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 970da50d56d5..d65ba937664b 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -40,6 +40,11 @@ public class DatastoreServiceTest { .setProperty("bool", BOOL_VALUE) .setProperty("list", LIST_VALUE1) .build(); + private static final PartialEntity PARTIAL_ENTITY2 = PartialEntity.builder(PARTIAL_ENTITY1) + .removeProperty("str") + .setBooleanProperty("bool", true) + .setListProperty("list", LIST_VALUE1.get()) + .build(); private static final Entity ENTITY1 = Entity.builder(KEY1) .setProperty("str", STR_VALUE) .setProperty("bool", BOOL_VALUE) @@ -49,13 +54,14 @@ public class DatastoreServiceTest { private static final Entity ENTITY2 = Entity.builder(ENTITY1) .key(KEY2) .removeProperty("str") - .setProperty("null", NULL_VALUE) + .setNullProperty("null") .build(); private static final Entity ENTITY3 = Entity.builder(ENTITY1) .key(KEY3) .removeProperty("str") .setProperty("null", NULL_VALUE) - .setProperty("partial2", new PartialEntityValue(ENTITY1)) + .setPartialEntityProperty("partial1", PARTIAL_ENTITY2) + .setPartialEntityProperty("partial2", ENTITY2) .build(); private DatastoreServiceOptions options; @@ -169,23 +175,24 @@ public void testAllocateId() { @Test public void testAllocateIdArray() { KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1); - PartialKey pKey1 = keyBuilder.build(); - PartialKey pKey2 = keyBuilder.kind(KIND2).addAncestor(KIND1, 10).build(); + PartialKey partialKey1 = keyBuilder.build(); + PartialKey partialKey2 = keyBuilder.kind(KIND2).addAncestor(KIND1, 10).build(); Key key3 = keyBuilder.build("name"); Key key4 = keyBuilder.build(1); - Iterator result = datastore.allocateId(pKey1, pKey2, key3, key4, pKey1, key3); + Iterator result = + datastore.allocateId(partialKey1, partialKey2, key3, key4, partialKey1, key3); Map map = new HashMap<>(); int count = 0; while (result.hasNext()) { map.put(++count, result.next()); } assertEquals(6, map.size()); - assertEquals(pKey1.newKey(map.get(1).id()), map.get(1)); - assertEquals(pKey1.newKey(map.get(5).id()), map.get(5)); - assertEquals(pKey2.newKey(map.get(2).id()), map.get(2)); - assertEquals(key3.builder().id(map.get(3).id()).build(), map.get(3)); - assertEquals(key3.builder().id(map.get(6).id()).build(), map.get(6)); - assertEquals(key4.builder().id(map.get(4).id()).build(), map.get(4)); + assertEquals(partialKey1.newKey(map.get(1).id()), map.get(1)); + assertEquals(partialKey1.newKey(map.get(5).id()), map.get(5)); + assertEquals(partialKey2.newKey(map.get(2).id()), map.get(2)); + assertEquals(Key.builder(key3).id(map.get(3).id()).build(), map.get(3)); + assertEquals(Key.builder(key3).id(map.get(6).id()).build(), map.get(6)); + assertEquals(Key.builder(key4).id(map.get(4).id()).build(), map.get(4)); } @Test @@ -194,24 +201,43 @@ public void testGet() { assertNull(entity); entity = datastore.get(KEY1); + assertEquals(ENTITY1, entity); StringValue value1 = entity.property("str"); BooleanValue value2 = entity.property("bool"); - PartialEntityValue value3 = entity.property("partial1"); + ListValue value3 = entity.property("list"); assertEquals(value1, STR_VALUE); assertEquals(value2, BOOL_VALUE); - assertEquals(value3, new PartialEntityValue(PARTIAL_ENTITY1)); - assertEquals(3, entity.propertyNames().size()); - assertTrue(entity.propertyNames().contains("str")); - assertTrue(entity.propertyNames().contains("bool")); + assertEquals(value3, LIST_VALUE2); + assertEquals(PARTIAL_ENTITY1, entity.partialEntityProperty("partial1")); + assertEquals(4, entity.propertyNames().size()); assertFalse(entity.hasProperty("bla")); } @Test public void testGetArray() { - Iterator result = datastore.get(KEY1, KEY1.builder().name("bla").build(), KEY2); + datastore.put(ENTITY3); + Iterator result = + datastore.get(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3); assertEquals(ENTITY1, result.next()); assertNull(result.next()); assertEquals(ENTITY2, result.next()); + Entity entity3 = result.next(); + assertEquals(ENTITY3, entity3); + assertTrue(entity3.isNullProperty("null")); + assertEquals(false, entity3.booleanProperty("bool")); + assertEquals(LIST_VALUE2.get(), entity3.listProperty("list")); + PartialEntity partial1 = entity3.partialEntityProperty("partial1"); + Entity partial2 = (Entity) entity3.partialEntityProperty("partial2"); + assertEquals(partial1, PARTIAL_ENTITY2); + assertEquals(partial2, ENTITY2); + assertEquals(Value.Type.BOOLEAN, entity3.propertyType("bool")); + assertEquals(5, entity3.propertyNames().size()); + assertFalse(entity3.hasProperty("bla")); + try { + entity3.stringProperty("str"); + } catch (DatastoreServiceException expected) { + // expected - no such property + } assertFalse(result.hasNext()); } @@ -297,6 +323,6 @@ public void testNewKeyBuilder() { assertEquals(PartialKey.builder(PARTIAL_KEY1).kind(KIND2).build(), datastore.newKeyBuilder(KIND2).build()); assertEquals(KEY1, keyBuilder.build("name")); - assertEquals(KEY1.builder().id(2).build(), keyBuilder.build(2)); + assertEquals(Key.builder(KEY1).id(2).build(), keyBuilder.build(2)); } } diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index a7b7c87baf7f..fc8e0be0d12e 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -96,6 +96,7 @@ public void testTypes() throws Exception { Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2, ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; for (Object obj : types) { + System.out.println("koko->" + obj); Object copy = serialiazeAndDeserialize(obj); assertEquals(obj, obj); assertEquals(obj, copy); From 63c28d442d4077051db18e84e059a7a8c7488e02 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 9 Dec 2014 12:11:59 -0800 Subject: [PATCH 040/732] work in progress --- .classpath | 3 + .../java/com/google/gcloud/AuthConfig.java | 2 +- .../google/gcloud/datastore/BaseEntity.java | 30 +++------ .../com/google/gcloud/datastore/BaseKey.java | 18 ----- .../com/google/gcloud/datastore/Blob.java | 19 ++++-- .../com/google/gcloud/datastore/Cursor.java | 11 +++- .../google/gcloud/datastore/EntityValue.java | 65 +++++++++++++++++++ .../java/com/google/gcloud/datastore/Key.java | 11 ++-- .../gcloud/datastore/PartialEntity.java | 2 +- .../gcloud/datastore/PartialEntityValue.java | 58 ----------------- .../google/gcloud/datastore/PartialKey.java | 8 --- .../google/gcloud/datastore/Serializable.java | 17 ----- .../datastore/DatastoreServiceTest.java | 12 ++-- .../gcloud/datastore/SerializationTest.java | 14 ++-- 14 files changed, 122 insertions(+), 148 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/EntityValue.java delete mode 100644 src/main/java/com/google/gcloud/datastore/PartialEntityValue.java diff --git a/.classpath b/.classpath index 9ed6ee4d0713..70f23326e0d2 100644 --- a/.classpath +++ b/.classpath @@ -21,6 +21,9 @@ + + + diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java index 117db5434a00..ddae5ffa1c72 100644 --- a/src/main/java/com/google/gcloud/AuthConfig.java +++ b/src/main/java/com/google/gcloud/AuthConfig.java @@ -8,7 +8,7 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.jackson.JacksonFactory; -import com.google.api.client.repackaged.com.google.common.base.Preconditions; +import com.google.common.base.Preconditions; import java.io.IOException; import java.security.GeneralSecurityException; diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java index 7aae77d360e2..b8a823ca01aa 100644 --- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java +++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java @@ -76,7 +76,7 @@ public B setBooleanProperty(String name, boolean value) { return self(); } - public B setDateAndTimeProperty(String name, DateTime value) { + public B setDateTimeProperty(String name, DateTime value) { properties.put(name, new DateTimeValue(value)); return self(); } @@ -86,8 +86,8 @@ public B setKeyProperty(String name, Key value) { return self(); } - public B setPartialEntityProperty(String name, PartialEntity value) { - properties.put(name, new PartialEntityValue(value)); + public B setEntityProperty(String name, PartialEntity value) { + properties.put(name, new EntityValue(value)); return self(); } @@ -162,7 +162,7 @@ public boolean booleanProperty(String name) { return ((BooleanValue) property(name)).get(); } - public DateTime dateAndTimeProperty(String name) { + public DateTime dateTimeProperty(String name) { return ((DateTimeValue) property(name)).get(); } @@ -170,8 +170,9 @@ public Key keyProperty(String name) { return ((KeyValue) property(name)).get(); } - public PartialEntity partialEntityProperty(String name) { - return ((PartialEntityValue) property(name)).get(); + @SuppressWarnings("unchecked") + public T entityProperty(String name) { + return (T) ((EntityValue) property(name)).get(); } public List> listProperty(String name) { @@ -183,9 +184,9 @@ public Blob blobProperty(String name) { } /** - * Returns the property's value as {@code RawValue}. + * Returns the property's value as a {@link RawValue}. */ - public RawValue rawValueProperty(String name) { + public RawValue asRawValueProperty(String name) { Value value = property(name); if (value instanceof RawValue) { return (RawValue) value; @@ -201,19 +202,6 @@ public Set propertyNames() { return properties; } - @Override - public int hashCode() { - return properties.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof BaseEntity)) { - return false; - } - return properties.equals(((BaseEntity) obj).properties); - } - @Override protected final DatastoreV1.Entity toPb() { DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java index 47859cd619d4..53d58effc1bc 100644 --- a/src/main/java/com/google/gcloud/datastore/BaseKey.java +++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java @@ -11,7 +11,6 @@ import java.util.LinkedList; import java.util.List; -import java.util.Objects; /** * Base class for keys. @@ -148,23 +147,6 @@ public String kind() { return kind; } - @Override - public int hashCode() { - return Objects.hash(dataset, namespace, ancestors, kind); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof BaseKey)) { - return false; - } - BaseKey other = (BaseKey) obj; - return Objects.equals(dataset, other.dataset) - && Objects.equals(namespace, other.namespace) - && Objects.equals(ancestors, other.ancestors) - && Objects.equals(kind, other.kind); - } - @Override protected DatastoreV1.Key toPb() { DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder(); diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java index e7e134815236..4edfdabe7ebf 100644 --- a/src/main/java/com/google/gcloud/datastore/Blob.java +++ b/src/main/java/com/google/gcloud/datastore/Blob.java @@ -1,8 +1,10 @@ package com.google.gcloud.datastore; -import static com.google.api.client.util.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; import com.google.protobuf.ByteString; import java.io.BufferedInputStream; @@ -10,7 +12,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.Objects; /** * A Google Cloud Datastore Blob. @@ -29,13 +30,21 @@ public final class Blob implements java.io.Serializable { Blob(ByteString byteString, boolean enforceLimits) { this.byteString = checkNotNull(byteString); if (enforceLimits) { - checkArgument(byteString.size() <= MAX_LENGTH, "May be a maximum of 1,000,000 bytes"); + checkArgument(byteString.size() <= MAX_LENGTH, "May be a maximum of %,d bytes", MAX_LENGTH); } } @Override public String toString() { - return byteString.toString(); + ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); + StringBuilder stBuilder = new StringBuilder(); + for (int i = 0; i < Math.min(256, byteString.size()); i++) { + stBuilder.append(String.format("%02x", byteString.byteAt(i))); + } + if (byteString.size() > 256) { + stBuilder.append("..."); + } + return toStringHelper.add("bytes", stBuilder.toString()).toString(); } @Override @@ -48,7 +57,7 @@ public boolean equals(Object obj) { if (!(obj instanceof Blob)) { return false; } - return Objects.equals(byteString, ((Blob) obj).byteString); + return byteString.equals(((Blob) obj).byteString); } /** diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java index e1163da38150..5b3ba6981af3 100644 --- a/src/main/java/com/google/gcloud/datastore/Cursor.java +++ b/src/main/java/com/google/gcloud/datastore/Cursor.java @@ -1,8 +1,10 @@ package com.google.gcloud.datastore; -import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkNotNull; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; import com.google.protobuf.ByteString; import java.io.Serializable; @@ -41,7 +43,12 @@ public boolean equals(Object obj) { @Override public String toString() { - return toPb().toString(); + ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); + StringBuilder stBuilder = new StringBuilder(); + for (byte b : bytes) { + stBuilder.append(String.format("%02x", b)); + } + return toStringHelper.add("bytes", stBuilder.toString()).toString(); } /** diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java new file mode 100644 index 000000000000..22c0932cc7af --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java @@ -0,0 +1,65 @@ +package com.google.gcloud.datastore; + +import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; + +import com.google.api.services.datastore.DatastoreV1; + +public class EntityValue extends Value { + + private static final long serialVersionUID = -5461475706792576395L; + + static final BaseMarshaller MARSHALLER = + new BaseMarshaller() { + + @Override + public int getProtoFieldId() { + return ENTITY_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(PartialEntity value) { + return builder(value); + } + + @Override + protected PartialEntity getValue(DatastoreV1.Value from) { + return PartialEntity.fromPb(from.getEntityValue()); + } + + @Override + protected void setValue(EntityValue from, DatastoreV1.Value.Builder to) { + to.setEntityValue(from.get().toPb()); + } + }; + + public static final class Builder extends + Value.BaseBuilder { + + private Builder() { + super(Type.ENTITY); + } + + @Override + public Builder indexed(boolean indexed) { + DatastoreServiceException.throwInvalidRequest("EntityValue can't specify index"); + return this; + } + + @Override + public EntityValue build() { + return new EntityValue(this); + } + } + + public EntityValue(PartialEntity entity) { + this(builder(entity)); + } + + private EntityValue(Builder builder) { + super(builder); + } + + public static Builder builder(PartialEntity entity) { + return new Builder().set(entity); + } +} diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index f569ed811194..d518afca8cdc 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -146,7 +146,7 @@ public static Key fromUrlSafe(String urlSafe) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), name, id); + return Objects.hash(dataset(), namespace(), ancestors(), kind(), name, id); } @Override @@ -155,9 +155,12 @@ public boolean equals(Object obj) { return false; } Key other = (Key) obj; - return Objects.equals(name, other.name) - && Objects.equals(id, other.id) - && super.equals(obj); + return Objects.equals(dataset(), other.dataset()) + && Objects.equals(namespace(), other.namespace()) + && Objects.equals(ancestors(), other.ancestors()) + && Objects.equals(kind(), other.kind()) + && Objects.equals(name, other.name) + && Objects.equals(id, other.id); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index 0796b2c9f5a1..dfc23c4dc1be 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -70,7 +70,7 @@ public boolean equals(Object obj) { } PartialEntity other = (PartialEntity) obj; return Objects.equals(key, other.key) - && super.equals(obj); + && Objects.equals(properties(), other.properties()); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java b/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java deleted file mode 100644 index db076a3305a0..000000000000 --- a/src/main/java/com/google/gcloud/datastore/PartialEntityValue.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.google.gcloud.datastore; - -import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; - -import com.google.api.services.datastore.DatastoreV1; - -public class PartialEntityValue extends - Value { - - private static final long serialVersionUID = -5461475706792576395L; - - static final BaseMarshaller MARSHALLER = - new BaseMarshaller() { - - @Override - public int getProtoFieldId() { - return ENTITY_VALUE_FIELD_NUMBER; - } - - @Override - public Builder newBuilder(PartialEntity value) { - return new Builder(value); - } - - @Override - protected PartialEntity getValue(DatastoreV1.Value from) { - return PartialEntity.fromPb(from.getEntityValue()); - } - - @Override - protected void setValue(PartialEntityValue from, DatastoreV1.Value.Builder to) { - to.setEntityValue(from.get().toPb()); - } - }; - - public static final class Builder extends - Value.BaseBuilder { - - public Builder(PartialEntity entity) { - super(Type.PARTIAL_ENTITY); - indexed(false); - set(entity); - } - - @Override - public PartialEntityValue build() { - return new PartialEntityValue(this); - } - } - - public PartialEntityValue(PartialEntity entity) { - this(new Builder(entity)); - } - - PartialEntityValue(Builder builder) { - super(builder); - } -} diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 129dbdf6e87d..7a27c2fd4b09 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -46,14 +46,6 @@ public Key newKey(long id) { return new Key(dataset(), namespace(), ImmutableList.copyOf(ancestors()), kind(), id); } - @Override - public boolean equals(Object obj) { - if (!(obj instanceof PartialKey)) { - return false; - } - return super.equals(obj); - } - @Override protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { return fromPb(DatastoreV1.Key.parseFrom(bytesPb)); diff --git a/src/main/java/com/google/gcloud/datastore/Serializable.java b/src/main/java/com/google/gcloud/datastore/Serializable.java index e9964b6d0e4a..c9f6b7984512 100644 --- a/src/main/java/com/google/gcloud/datastore/Serializable.java +++ b/src/main/java/com/google/gcloud/datastore/Serializable.java @@ -15,23 +15,6 @@ abstract class Serializable implements java.io.Seria private transient byte[] bytesPb; // only for deserialization - - @Override - public int hashCode() { - return toPb().hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (!getClass().isInstance(obj)) { - return false; - } - return toPb().equals(((Serializable) obj).toPb()); - } - @Override public String toString() { return toPb().toString(); diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index d65ba937664b..40397e35c47c 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -48,7 +48,7 @@ public class DatastoreServiceTest { private static final Entity ENTITY1 = Entity.builder(KEY1) .setProperty("str", STR_VALUE) .setProperty("bool", BOOL_VALUE) - .setProperty("partial1", new PartialEntityValue(PARTIAL_ENTITY1)) + .setProperty("partial1", new EntityValue(PARTIAL_ENTITY1)) .setProperty("list", LIST_VALUE2) .build(); private static final Entity ENTITY2 = Entity.builder(ENTITY1) @@ -60,8 +60,8 @@ public class DatastoreServiceTest { .key(KEY3) .removeProperty("str") .setProperty("null", NULL_VALUE) - .setPartialEntityProperty("partial1", PARTIAL_ENTITY2) - .setPartialEntityProperty("partial2", ENTITY2) + .setEntityProperty("partial1", PARTIAL_ENTITY2) + .setEntityProperty("partial2", ENTITY2) .build(); private DatastoreServiceOptions options; @@ -208,7 +208,7 @@ public void testGet() { assertEquals(value1, STR_VALUE); assertEquals(value2, BOOL_VALUE); assertEquals(value3, LIST_VALUE2); - assertEquals(PARTIAL_ENTITY1, entity.partialEntityProperty("partial1")); + assertEquals(PARTIAL_ENTITY1, entity.entityProperty("partial1")); assertEquals(4, entity.propertyNames().size()); assertFalse(entity.hasProperty("bla")); } @@ -226,8 +226,8 @@ public void testGetArray() { assertTrue(entity3.isNullProperty("null")); assertEquals(false, entity3.booleanProperty("bool")); assertEquals(LIST_VALUE2.get(), entity3.listProperty("list")); - PartialEntity partial1 = entity3.partialEntityProperty("partial1"); - Entity partial2 = (Entity) entity3.partialEntityProperty("partial2"); + PartialEntity partial1 = entity3.entityProperty("partial1"); + Entity partial2 = (Entity) entity3.entityProperty("partial2"); assertEquals(partial1, PARTIAL_ENTITY2); assertEquals(partial2, ENTITY2); assertEquals(Value.Type.BOOLEAN, entity3.propertyType("bool")); diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index fc8e0be0d12e..96b6d09240b6 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -49,12 +49,12 @@ public class SerializationTest { .setProperty("p1", STRING_VALUE) .setProperty("p2", LongValue.builder(100).indexed(false).meaning(100).build()) .build(); - private static final PartialEntityValue EMBEDDED_ENTITY_VALUE1 = - new PartialEntityValue(EMBEDDED_ENTITY1); - private static final PartialEntityValue EMBEDDED_ENTITY_VALUE2 = - new PartialEntityValue(EMBEDDED_ENTITY2); - private static final PartialEntityValue EMBEDDED_ENTITY_VALUE3 = - new PartialEntityValue(EMBEDDED_ENTITY3); + private static final EntityValue EMBEDDED_ENTITY_VALUE1 = + new EntityValue(EMBEDDED_ENTITY1); + private static final EntityValue EMBEDDED_ENTITY_VALUE2 = + new EntityValue(EMBEDDED_ENTITY2); + private static final EntityValue EMBEDDED_ENTITY_VALUE3 = + new EntityValue(EMBEDDED_ENTITY3); private static final ListValue LIST_VALUE = ListValue.builder() .addValue(NULL_VALUE) .addValue(STRING_VALUE) @@ -66,7 +66,7 @@ public class SerializationTest { .put(Type.NULL, NULL_VALUE) .put(Type.KEY, KEY_VALUE) .put(Type.STRING, STRING_VALUE) - .putAll(Type.PARTIAL_ENTITY, EMBEDDED_ENTITY_VALUE1, EMBEDDED_ENTITY_VALUE2, + .putAll(Type.ENTITY, EMBEDDED_ENTITY_VALUE1, EMBEDDED_ENTITY_VALUE2, EMBEDDED_ENTITY_VALUE3) .put(Type.LIST, LIST_VALUE) .put(Type.LONG, LONG_VALUE) From 28c5d62eb96075a563e859eb6081967f2f04e546 Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 9 Dec 2014 14:18:27 -0800 Subject: [PATCH 041/732] change Value generics --- checkstyle.xml | 165 ++++++++++++++++++ .../google/gcloud/datastore/BaseEntity.java | 24 +-- .../com/google/gcloud/datastore/BaseKey.java | 30 ++-- .../google/gcloud/datastore/BlobValue.java | 7 +- .../google/gcloud/datastore/BooleanValue.java | 7 +- .../gcloud/datastore/DateTimeValue.java | 8 +- .../google/gcloud/datastore/DoubleValue.java | 7 +- .../com/google/gcloud/datastore/Entity.java | 4 +- .../google/gcloud/datastore/EntityValue.java | 14 +- .../java/com/google/gcloud/datastore/Key.java | 36 ++-- .../google/gcloud/datastore/KeyBuilder.java | 2 +- .../com/google/gcloud/datastore/KeyValue.java | 7 +- .../google/gcloud/datastore/ListValue.java | 42 +++-- .../google/gcloud/datastore/LongValue.java | 7 +- .../google/gcloud/datastore/NullValue.java | 7 +- .../gcloud/datastore/PartialEntity.java | 8 +- .../google/gcloud/datastore/PartialKey.java | 44 +++-- .../{KeyPathElement.java => PathElement.java} | 20 +-- .../com/google/gcloud/datastore/RawValue.java | 7 +- .../google/gcloud/datastore/StringValue.java | 7 +- .../com/google/gcloud/datastore/Value.java | 55 +++--- .../gcloud/datastore/SerializationTest.java | 5 +- 22 files changed, 366 insertions(+), 147 deletions(-) create mode 100644 checkstyle.xml rename src/main/java/com/google/gcloud/datastore/{KeyPathElement.java => PathElement.java} (76%) diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 000000000000..a3c9eecc8e46 --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,165 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java index b8a823ca01aa..41a6674cbcee 100644 --- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java +++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java @@ -17,11 +17,11 @@ abstract class BaseEntity extends Serializable { private static final long serialVersionUID = 8175618724683792766L; - private final transient ImmutableSortedMap> properties; + private final transient ImmutableSortedMap> properties; protected abstract static class Builder> { - private final Map> properties; + private final Map> properties; protected Builder() { properties = new HashMap<>(); @@ -46,7 +46,7 @@ public B removeProperty(String name) { return self(); } - public B setProperty(String name, Value value) { + public B setProperty(String name, Value value) { properties.put(name, value); return self(); } @@ -91,12 +91,12 @@ public B setEntityProperty(String name, PartialEntity value) { return self(); } - public B setListProperty(String name, List> values) { + public B setListProperty(String name, List> values) { properties.put(name, new ListValue(values)); return self(); } - public B setListProperty(String name, Value... value) { + public B setListProperty(String name, Value... value) { properties.put(name, new ListValue(Arrays.asList(value))); return self(); } @@ -110,10 +110,10 @@ public E build() { return build(ImmutableSortedMap.copyOf(properties)); } - protected abstract E build(ImmutableSortedMap> properties); + protected abstract E build(ImmutableSortedMap> properties); } - protected BaseEntity(ImmutableSortedMap> properties) { + protected BaseEntity(ImmutableSortedMap> properties) { this.properties = properties; } @@ -129,7 +129,7 @@ public boolean hasProperty(String name) { * * @throws DatastoreServiceException if not such property. */ - public > V property(String name) { + public > V property(String name) { @SuppressWarnings("unchecked") V property = (V) properties.get(name); if (property == null) { @@ -175,7 +175,7 @@ public T entityProperty(String name) { return (T) ((EntityValue) property(name)).get(); } - public List> listProperty(String name) { + public List> listProperty(String name) { return ((ListValue) property(name)).get(); } @@ -187,7 +187,7 @@ public Blob blobProperty(String name) { * Returns the property's value as a {@link RawValue}. */ public RawValue asRawValueProperty(String name) { - Value value = property(name); + Value value = property(name); if (value instanceof RawValue) { return (RawValue) value; } @@ -198,14 +198,14 @@ public Set propertyNames() { return properties.keySet(); } - ImmutableSortedMap> properties() { + ImmutableSortedMap> properties() { return properties; } @Override protected final DatastoreV1.Entity toPb() { DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder(); - for (Map.Entry> entry : properties.entrySet()) { + for (Map.Entry> entry : properties.entrySet()) { DatastoreV1.Property.Builder propertyPb = DatastoreV1.Property.newBuilder(); propertyPb.setName(entry.getKey()); propertyPb.setValue(entry.getValue().toPb()); diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java index 53d58effc1bc..09edf60afeac 100644 --- a/src/main/java/com/google/gcloud/datastore/BaseKey.java +++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java @@ -21,7 +21,7 @@ abstract class BaseKey extends Serializable { private final transient String dataset; private final transient String namespace; - private final transient ImmutableList ancestors; + private final transient ImmutableList ancestors; private final transient String kind; abstract static class Builder> { @@ -29,7 +29,7 @@ abstract static class Builder> { private String dataset; private String namespace; private String kind; - private final List ancestors; + private final List ancestors; private static final int MAX_PATH = 100; @@ -53,26 +53,26 @@ protected B self() { public B addAncestor(String kind, long id) { checkArgument(id != 0, "id must not be equal to zero"); - return addAncestor(new KeyPathElement(kind, id)); + return addAncestor(new PathElement(kind, id)); } public B addAncestor(String kind, String name) { checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null"); checkArgument(name.length() <= 500, "name must not exceed 500 characters"); - return addAncestor(new KeyPathElement(kind, name)); + return addAncestor(new PathElement(kind, name)); } - public B addAncestor(KeyPathElement... ancestor) { + public B addAncestor(PathElement... ancestor) { Preconditions.checkState(ancestors.size() + ancestor.length <= MAX_PATH, "path can have at most 100 elements"); - for (KeyPathElement pathElement : ancestor) { + for (PathElement pathElement : ancestor) { ancestors.add(pathElement); } return self(); } - public B addAncestors(Iterable ancestors) { - for (KeyPathElement pathElement : ancestors) { + public B addAncestors(Iterable ancestors) { + for (PathElement pathElement : ancestors) { addAncestor(pathElement); } return self(); @@ -109,10 +109,10 @@ public K build() { } protected abstract K build( - String dataset, String namespace, ImmutableList ancestors, String kind); + String dataset, String namespace, ImmutableList ancestors, String kind); } - BaseKey(String dataset, String namespace, ImmutableList ancestors, String kind) { + BaseKey(String dataset, String namespace, ImmutableList ancestors, String kind) { this.dataset = dataset; this.namespace = namespace; this.ancestors = ancestors; @@ -136,7 +136,7 @@ public String namespace() { /** * Returns an immutable list with the key's ancestors. */ - public List ancestors() { + public List ancestors() { return ancestors; } @@ -160,16 +160,12 @@ protected DatastoreV1.Key toPb() { if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) { keyPb.setPartitionId(partitionIdPb.build()); } - for (KeyPathElement pathEntry : ancestors) { + for (PathElement pathEntry : ancestors) { keyPb.addPathElement(pathEntry.toPb()); } addLeaf(keyPb); return keyPb.build(); } - protected void addLeaf(DatastoreV1.Key.Builder keyPb) { - if (kind != null) { - keyPb.addPathElement(new KeyPathElement(kind).toPb()); - } - } + protected abstract void addLeaf(DatastoreV1.Key.Builder keyPb); } diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java index 0b3a3fed9129..8d369c9eb447 100644 --- a/src/main/java/com/google/gcloud/datastore/BlobValue.java +++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java @@ -4,7 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class BlobValue extends Value { +public final class BlobValue extends Value { private static final long serialVersionUID = -5096238337676649540L; @@ -52,6 +52,11 @@ private BlobValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(Blob blob) { return new Builder().set(blob); } diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/src/main/java/com/google/gcloud/datastore/BooleanValue.java index 533ee14d5b0d..f6323bb3f794 100644 --- a/src/main/java/com/google/gcloud/datastore/BooleanValue.java +++ b/src/main/java/com/google/gcloud/datastore/BooleanValue.java @@ -4,7 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class BooleanValue extends Value { +public final class BooleanValue extends Value { private static final long serialVersionUID = -542649497897250340L; @@ -52,6 +52,11 @@ private BooleanValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(boolean value) { return new Builder().set(value); } diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java index 85ac6a853779..a178c71a9665 100644 --- a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java +++ b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java @@ -4,8 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class DateTimeValue - extends Value { +public final class DateTimeValue extends Value { private static final long serialVersionUID = -5096238337676649540L; @@ -54,6 +53,11 @@ private DateTimeValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(DateTime dateTime) { return new Builder().set(dateTime); } diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/src/main/java/com/google/gcloud/datastore/DoubleValue.java index b2f4e264c723..749a2a9b7bd5 100644 --- a/src/main/java/com/google/gcloud/datastore/DoubleValue.java +++ b/src/main/java/com/google/gcloud/datastore/DoubleValue.java @@ -4,7 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class DoubleValue extends Value { +public final class DoubleValue extends Value { private static final long serialVersionUID = -5096238337676649540L; @@ -52,6 +52,11 @@ private DoubleValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(double value) { return new Builder().set(value); } diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java index f497cb6c7d67..5262fcccd8c1 100644 --- a/src/main/java/com/google/gcloud/datastore/Entity.java +++ b/src/main/java/com/google/gcloud/datastore/Entity.java @@ -39,12 +39,12 @@ public Builder key(Key key) { } @Override - protected Entity build(ImmutableSortedMap> properties) { + protected Entity build(ImmutableSortedMap> properties) { return new Entity(key, properties); } } - Entity(Key key, ImmutableSortedMap> properties) { + Entity(Key key, ImmutableSortedMap> properties) { super(key, properties); } diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java index 22c0932cc7af..3b89d81bb4be 100644 --- a/src/main/java/com/google/gcloud/datastore/EntityValue.java +++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java @@ -3,8 +3,9 @@ import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER; import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.Preconditions; -public class EntityValue extends Value { +public class EntityValue extends Value { private static final long serialVersionUID = -5461475706792576395L; @@ -41,8 +42,8 @@ private Builder() { @Override public Builder indexed(boolean indexed) { - DatastoreServiceException.throwInvalidRequest("EntityValue can't specify index"); - return this; + Preconditions.checkArgument(!indexed, "EntityValue can't be indexed"); + return super.indexed(indexed); } @Override @@ -59,7 +60,12 @@ private EntityValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(PartialEntity entity) { - return new Builder().set(entity); + return new Builder().set(entity).indexed(false); } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index d518afca8cdc..b8a027069a85 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -24,8 +24,7 @@ public final class Key extends PartialKey { private static final long serialVersionUID = 3160994559785491356L; - private final transient String name; - private final transient Long id; + private final transient PathElement leaf; public static final class Builder extends BaseKey.Builder { @@ -64,7 +63,7 @@ public Builder id(long id) { } @Override - protected Key build(String dataset, String namespace, ImmutableList ancestors, + protected Key build(String dataset, String namespace, ImmutableList ancestors, String kind) { if (id == null) { return new Key(dataset, namespace, ancestors, kind, name); @@ -73,47 +72,45 @@ protected Key build(String dataset, String namespace, ImmutableList ancestors, + Key(String dataset, String namespace, ImmutableList ancestors, String kind, String name) { super(dataset, namespace, ancestors, kind); - this.name = name; - id = null; + leaf = new PathElement(kind, name); } - Key(String dataset, String namespace, ImmutableList ancestors, + Key(String dataset, String namespace, ImmutableList ancestors, String kind, long id) { super(dataset, namespace, ancestors, kind); - this.id = id; - name = null; + leaf = new PathElement(kind, id); } public boolean hasId() { - return id != null; + return leaf.hasId(); } /** * Returns the key's id or {@code null} if it has a name instead. */ public Long id() { - return id; + return leaf.id(); } public boolean hasName() { - return name != null; + return leaf.hasName(); } /** * Returns the key's name or {@code null} if it has an id instead. */ public String name() { - return name; + return leaf.name(); } /** * Returns the key's id (as {@link #Long}) or name (as {@link String}). */ public Object nameOrId() { - return hasId() ? id : name; + return leaf.nameOrId(); } /** @@ -146,7 +143,7 @@ public static Key fromUrlSafe(String urlSafe) { @Override public int hashCode() { - return Objects.hash(dataset(), namespace(), ancestors(), kind(), name, id); + return Objects.hash(dataset(), namespace(), ancestors(), kind(), leaf); } @Override @@ -159,14 +156,11 @@ public boolean equals(Object obj) { && Objects.equals(namespace(), other.namespace()) && Objects.equals(ancestors(), other.ancestors()) && Objects.equals(kind(), other.kind()) - && Objects.equals(name, other.name) - && Objects.equals(id, other.id); + && Objects.equals(leaf, other.leaf); } @Override protected void addLeaf(DatastoreV1.Key.Builder keyPb) { - KeyPathElement leaf = - hasId() ? new KeyPathElement(kind(), id) : new KeyPathElement(kind(), name); keyPb.addPathElement(leaf.toPb()); } @@ -209,9 +203,9 @@ private static void addParentToBuilder(Key parent, Builder builder) { builder.namespace(parent.namespace()); builder.addAncestors(parent.ancestors()); if (parent.hasId()) { - builder.addAncestor(new KeyPathElement(parent.kind(), parent.id())); + builder.addAncestor(new PathElement(parent.kind(), parent.id())); } else { - builder.addAncestor(new KeyPathElement(parent.kind(), parent.name())); + builder.addAncestor(new PathElement(parent.kind(), parent.name())); } } } diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java index 855c09d3358f..6860063124d0 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java +++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java @@ -23,7 +23,7 @@ public KeyBuilder(DatastoreService service, String kind) { @Override protected PartialKey build(String dataset, String namespace, - ImmutableList ancestors, String kind) { + ImmutableList ancestors, String kind) { return new PartialKey(dataset, namespace, ancestors, kind); } diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java index 8a6db4ff74a4..b9aab3134a86 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyValue.java +++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java @@ -4,7 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class KeyValue extends Value { +public final class KeyValue extends Value { private static final long serialVersionUID = -1318353707326704821L; @@ -52,6 +52,11 @@ private KeyValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(Key key) { return new Builder().set(key); } diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java index 883b46f0fc28..cd33c96dcff0 100644 --- a/src/main/java/com/google/gcloud/datastore/ListValue.java +++ b/src/main/java/com/google/gcloud/datastore/ListValue.java @@ -9,13 +9,12 @@ import java.util.ArrayList; import java.util.List; -public final class ListValue extends - Value>, ListValue, ListValue.Builder> { +public final class ListValue extends Value>> { private static final long serialVersionUID = -5461475706792576395L; - static final BaseMarshaller>, ListValue, Builder> MARSHALLER = - new BaseMarshaller>, ListValue, Builder>() { + static final BaseMarshaller>, ListValue, Builder> MARSHALLER = + new BaseMarshaller>, ListValue, Builder>() { @Override public int getProtoFieldId() { @@ -23,13 +22,13 @@ public int getProtoFieldId() { } @Override - public Builder newBuilder(List> values) { + public Builder newBuilder(List> values) { return builder().set(values); } @Override - protected List> getValue(DatastoreV1.Value from) { - List> properties = new ArrayList<>(from.getListValueCount()); + protected List> getValue(DatastoreV1.Value from) { + List> properties = new ArrayList<>(from.getListValueCount()); for (DatastoreV1.Value valuePb : from.getListValueList()) { properties.add(Value.fromPb(valuePb)); } @@ -38,30 +37,30 @@ public Builder newBuilder(List> values) { @Override protected void setValue(ListValue from, DatastoreV1.Value.Builder to) { - for (Value property : from.get()) { + for (Value property : from.get()) { to.addListValue(property.toPb()); } } }; public static final class Builder extends - Value.BaseBuilder>, ListValue, Builder> { + Value.BaseBuilder>, ListValue, Builder> { - private ImmutableList.Builder> listBuilder = ImmutableList.builder(); + private ImmutableList.Builder> listBuilder = ImmutableList.builder(); private Builder() { super(Type.LIST); } - public Builder addValue(Value value) { + public Builder addValue(Value value) { Preconditions.checkArgument(value.type() != Type.LIST, "Cannot contain another list"); listBuilder.add(value); return this; } - public Builder addValue(Value first, Value... other) { + public Builder addValue(Value first, Value... other) { addValue(first); - for (Value value : other) { + for (Value value : other) { addValue(value); } return this; @@ -79,16 +78,16 @@ public Builder indexed(boolean indexed) { * @see com.google.gcloud.datastore.Value.BaseBuilder#set(java.lang.Object) */ @Override - public Builder set(List> properties) { - listBuilder = ImmutableList.>builder(); - for (Value property : properties) { + public Builder set(List> properties) { + listBuilder = ImmutableList.>builder(); + for (Value property : properties) { addValue(property); } return this; } @Override - public List> get() { + public List> get() { return listBuilder.build(); } @@ -99,11 +98,11 @@ public ListValue build() { } } - public ListValue(List> properties) { + public ListValue(List> properties) { this(builder().set(properties)); } - public ListValue(Value first, Value... other) { + public ListValue(Value first, Value... other) { this(new Builder().addValue(first, other)); } @@ -111,6 +110,11 @@ private ListValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder() { return new Builder(); } diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/src/main/java/com/google/gcloud/datastore/LongValue.java index 97b789e344bd..9107223c7af4 100644 --- a/src/main/java/com/google/gcloud/datastore/LongValue.java +++ b/src/main/java/com/google/gcloud/datastore/LongValue.java @@ -4,7 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class LongValue extends Value { +public final class LongValue extends Value { private static final long serialVersionUID = -8552854340400546861L; @@ -52,6 +52,11 @@ private LongValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(long value) { return new Builder().set(value); } diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java index bb2c084522ae..cf4f463095d0 100644 --- a/src/main/java/com/google/gcloud/datastore/NullValue.java +++ b/src/main/java/com/google/gcloud/datastore/NullValue.java @@ -4,7 +4,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class NullValue extends Value { +public final class NullValue extends Value { private static final long serialVersionUID = 8497300779013002270L; @@ -58,6 +58,11 @@ private NullValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder() { return new Builder(); } diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index dfc23c4dc1be..ccd754bc0712 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -37,18 +37,18 @@ public Builder key(PartialKey key) { } @Override - protected PartialEntity build(ImmutableSortedMap> properties) { + protected PartialEntity build(ImmutableSortedMap> properties) { return new PartialEntity(key, properties); } } - protected PartialEntity(PartialKey key, ImmutableSortedMap> properties) { + protected PartialEntity(PartialKey key, ImmutableSortedMap> properties) { super(properties); this.key = key; } public Entity newEntity(Key key) { - return new Entity(key, ImmutableSortedMap.>copyOf(properties())); + return new Entity(key, ImmutableSortedMap.>copyOf(properties())); } /** @@ -86,7 +86,7 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { } static PartialEntity fromPb(DatastoreV1.Entity entityPb) { - ImmutableSortedMap.Builder> properties = + ImmutableSortedMap.Builder> properties = ImmutableSortedMap.naturalOrder(); for (DatastoreV1.Property property : entityPb.getPropertyList()) { properties.put(property.getName(), Value.fromPb(property.getValue())); diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index 7a27c2fd4b09..e470a71c3376 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -1,11 +1,11 @@ package com.google.gcloud.datastore; import com.google.api.services.datastore.DatastoreV1; -import com.google.api.services.datastore.DatastoreV1.Key.PathElement; import com.google.common.collect.ImmutableList; import com.google.protobuf.InvalidProtocolBufferException; import java.util.List; +import java.util.Objects; /** * A partial key (without a name or id). @@ -28,13 +28,12 @@ private Builder(PartialKey copyFrom) { @Override protected PartialKey build(String dataset, String namespace, - ImmutableList ancestors, String kind) { + ImmutableList ancestors, String kind) { return new PartialKey(dataset, namespace, ancestors, kind); } } - PartialKey(String dataset, String namespace, ImmutableList ancestors, - String kind) { + PartialKey(String dataset, String namespace, ImmutableList ancestors, String kind) { super(dataset, namespace, ancestors, kind); } @@ -46,6 +45,29 @@ public Key newKey(long id) { return new Key(dataset(), namespace(), ImmutableList.copyOf(ancestors()), kind(), id); } + @Override + public int hashCode() { + return Objects.hash(dataset(), namespace(), ancestors(), kind()); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof PartialKey) || !PartialKey.class.equals(obj.getClass())) { + return false; + } + + PartialKey other = (PartialKey) obj; + return Objects.equals(dataset(), other.dataset()) + && Objects.equals(namespace(), other.namespace()) + && Objects.equals(ancestors(), other.ancestors()) + && Objects.equals(kind(), other.kind()); + } + + @Override + protected void addLeaf(DatastoreV1.Key.Builder keyPb) { + keyPb.addPathElement(new PathElement(kind()).toPb()); + } + @Override protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { return fromPb(DatastoreV1.Key.parseFrom(bytesPb)); @@ -63,15 +85,15 @@ static PartialKey fromPb(DatastoreV1.Key keyPb) { namespace = partitionIdPb.getNamespace(); } } - List pathElementsPb = keyPb.getPathElementList(); + List pathElementsPb = keyPb.getPathElementList(); if (pathElementsPb.isEmpty()) { - return new PartialKey(dataset, namespace, ImmutableList.of(), null); + return new PartialKey(dataset, namespace, ImmutableList.of(), null); } - ImmutableList.Builder pathBuilder = ImmutableList.builder(); + ImmutableList.Builder pathBuilder = ImmutableList.builder(); for (int i = 0; i < pathElementsPb.size() - 1; i++) { - pathBuilder.add(KeyPathElement.fromPb(pathElementsPb.get(i))); + pathBuilder.add(PathElement.fromPb(pathElementsPb.get(i))); } - PathElement leaf = pathElementsPb.get(pathElementsPb.size() - 1); + DatastoreV1.Key.PathElement leaf = pathElementsPb.get(pathElementsPb.size() - 1); String kind = leaf.getKind(); if (leaf.hasId()) { return new Key(dataset, namespace, pathBuilder.build(), kind, leaf.getId()); @@ -94,9 +116,9 @@ public static Builder builder(Key parent, String kind) { .namespace(parent.namespace()) .addAncestors(parent.ancestors()); if (parent.hasId()) { - builder.addAncestor(new KeyPathElement(parent.kind(), parent.id())); + builder.addAncestor(new PathElement(parent.kind(), parent.id())); } else { - builder.addAncestor(new KeyPathElement(parent.kind(), parent.name())); + builder.addAncestor(new PathElement(parent.kind(), parent.name())); } return builder; } diff --git a/src/main/java/com/google/gcloud/datastore/KeyPathElement.java b/src/main/java/com/google/gcloud/datastore/PathElement.java similarity index 76% rename from src/main/java/com/google/gcloud/datastore/KeyPathElement.java rename to src/main/java/com/google/gcloud/datastore/PathElement.java index 78b529ed6e7b..213c5a7bdae8 100644 --- a/src/main/java/com/google/gcloud/datastore/KeyPathElement.java +++ b/src/main/java/com/google/gcloud/datastore/PathElement.java @@ -8,7 +8,7 @@ /** * Represents a single element in a key's path. */ -public final class KeyPathElement extends Serializable { +public final class PathElement extends Serializable { private static final long serialVersionUID = -7968078857690784595L; @@ -16,17 +16,17 @@ public final class KeyPathElement extends Serializable { +public final class RawValue extends Value { private static final long serialVersionUID = -3359604598651897941L; @@ -50,6 +50,11 @@ private RawValue(Builder builder) { this(builder(valuePb)); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + static Builder builder(DatastoreV1.Value valuePb) { Builder builder = new Builder(); if (valuePb.hasIndexed()) { diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java index 4d35aa3e3e01..6434b6a86f40 100644 --- a/src/main/java/com/google/gcloud/datastore/StringValue.java +++ b/src/main/java/com/google/gcloud/datastore/StringValue.java @@ -5,7 +5,7 @@ import com.google.api.services.datastore.DatastoreV1; -public final class StringValue extends Value { +public final class StringValue extends Value { private static final long serialVersionUID = -3105699707394545523L; @@ -56,6 +56,11 @@ private StringValue(Builder builder) { super(builder); } + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + public static Builder builder(String value) { return new Builder().set(value); } diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java index d35820e38db7..448efe37ed31 100644 --- a/src/main/java/com/google/gcloud/datastore/Value.java +++ b/src/main/java/com/google/gcloud/datastore/Value.java @@ -4,7 +4,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.api.services.datastore.DatastoreV1; -import com.google.common.base.MoreObjects; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.InvalidProtocolBufferException; @@ -23,9 +22,7 @@ * @param

the type of this value * @param the type of this value's builder */ -public abstract class - Value, B extends Value.Builder> - extends Serializable { +public abstract class Value extends Serializable { private static final long serialVersionUID = -1899638277588872742L; private static final Map DESCRIPTOR_TO_TYPE_MAP = new HashMap<>(); @@ -53,9 +50,9 @@ public enum Type { STRING(StringValue.MARSHALLER, StringValue.MARSHALLER), /** - * Represents an {@link PartialEntity} value. + * Represents an entity ({@link PartialEntity} or {@link Entity}) value. */ - PARTIAL_ENTITY(PartialEntityValue.MARSHALLER, PartialEntityValue.MARSHALLER), + ENTITY(EntityValue.MARSHALLER, EntityValue.MARSHALLER), /** * Represents a {@code list} of {@link Value}s. @@ -101,8 +98,8 @@ public enum Type { @SuppressWarnings("rawtypes") private final BuilderFactory builderFactory; @SuppressWarnings("rawtypes") private final Marshaller marshaller; - , B extends Builder> - Type(Marshaller marshaller, BuilderFactory builderFactory) { + , B extends Builder> Type(Marshaller marshaller, + BuilderFactory builderFactory) { this.marshaller = marshaller; this.builderFactory = builderFactory; int fieldId = marshaller.getProtoFieldId(); @@ -111,23 +108,22 @@ public enum Type { } } - , B extends Builder> Marshaller - getMarshaller() { + , B extends Builder> Marshaller getMarshaller() { return marshaller; } - , B extends Builder> BuilderFactory + , B extends Builder> BuilderFactory getBuilderFactory() { return builderFactory; } } - interface BuilderFactory, B extends Builder> { + interface BuilderFactory, B extends Builder> { B newBuilder(V value); } - interface Builder, B extends Builder> { + interface Builder, B extends Builder> { Type getType(); @@ -148,7 +144,7 @@ interface Builder, B extends Builder> { P build(); } - interface Marshaller, B extends Builder> { + interface Marshaller, B extends Builder> { B fromProto(DatastoreV1.Value proto); @@ -157,7 +153,7 @@ interface Marshaller, B extends Builder> { int getProtoFieldId(); } - abstract static class BaseMarshaller, B extends Builder> + abstract static class BaseMarshaller, B extends Builder> implements Marshaller, BuilderFactory { @Override @@ -190,7 +186,7 @@ public final DatastoreV1.Value toProto(P value) { protected abstract void setValue(P from, DatastoreV1.Value.Builder to); } - abstract static class BaseBuilder, B extends BaseBuilder> + abstract static class BaseBuilder, B extends BaseBuilder> implements Builder { private final Type type; @@ -257,7 +253,7 @@ private B self() { public abstract P build(); } - Value(Builder builder) { +

, B extends BaseBuilder> Value(Builder builder) { type = builder.getType(); indexed = builder.getIndexed(); meaning = builder.getMeaning(); @@ -280,9 +276,8 @@ public final boolean hasIndexed() { return indexed != null; } - public final boolean indexed() { - // default indexed value is true - return MoreObjects.firstNonNull(indexed, Boolean.TRUE); + public final Boolean indexed() { + return indexed; } public final boolean hasMeaning() { @@ -297,20 +292,15 @@ public final V get() { return value; } - @SuppressWarnings("unchecked") - public final B toBuilder() { - BuilderFactory builderFactory = type().getBuilderFactory(); - B builder = builderFactory.newBuilder(get()); - return builder.mergeFrom((P) this); - } + public abstract Builder toBuilder(); @Override public int hashCode() { - return Objects.hash(type, indexed, meaning); + return Objects.hash(type, indexed, meaning, value); } - @SuppressWarnings("unchecked") @Override + @SuppressWarnings("unchecked") public boolean equals(Object obj) { if (obj == this) { return true; @@ -318,7 +308,7 @@ public boolean equals(Object obj) { if (!getClass().isInstance(obj)) { return false; } - Value other = (Value) obj; + Value other = (Value) obj; return Objects.equals(type, other.type) && Objects.equals(indexed, other.indexed) && Objects.equals(meaning, other.meaning) @@ -326,13 +316,12 @@ public boolean equals(Object obj) { } @Override - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked", "rawtypes"}) protected DatastoreV1.Value toPb() { - Marshaller marshaller = type().getMarshaller(); - return marshaller.toProto((P) this); + return type().getMarshaller().toProto((Value) this); } - static Value fromPb(DatastoreV1.Value proto) { + static Value fromPb(DatastoreV1.Value proto) { for (Entry entry : proto.getAllFields().entrySet()) { FieldDescriptor descriptor = entry.getKey(); if (descriptor.getName().endsWith("_value")) { diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 96b6d09240b6..92ad4a672460 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -80,8 +80,8 @@ public class SerializationTest { @Test public void testValues() throws Exception { for (Type type : Type.values()) { - for (Value value : typeToValues.get(type)) { - Value copy = serialiazeAndDeserialize(value); + for (Value value : typeToValues.get(type)) { + Value copy = serialiazeAndDeserialize(value); assertEquals(value, value); assertEquals(value, copy); assertNotSame(value, copy); @@ -96,7 +96,6 @@ public void testTypes() throws Exception { Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2, ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; for (Object obj : types) { - System.out.println("koko->" + obj); Object copy = serialiazeAndDeserialize(obj); assertEquals(obj, obj); assertEquals(obj, copy); From baf41a7b3a2b69d6762e8c76ccd788cb82f3466d Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 9 Dec 2014 14:23:53 -0800 Subject: [PATCH 042/732] fix package-info.java --- src/main/java/com/google/gcloud/datastore/package-info.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java index 60aaddb13a74..5b2b8b8e8db5 100644 --- a/src/main/java/com/google/gcloud/datastore/package-info.java +++ b/src/main/java/com/google/gcloud/datastore/package-info.java @@ -16,11 +16,11 @@ * datastore.put(entity); * } else { * boolean updated = entity.booleanProperty("updated"); - * if (!updated.get) { + * if (!updated) { * String[] name = entity.stringProperty("name").split(" "); - * entity = entity.builder() + * entity = Entity.builder(entity) * .setStringProperty("name", name[0]) - * .setProperty("last_name", StringProperty.builder(name[1]).indexed(false).build()) + * .setProperty("last_name", StringValue.builder(name[1]).indexed(false).build()) * .setBooleanProperty("updated", true) * .removeProperty("old_property") * .setDoubleProperty("new_property", 1.1) From 546cd049cbde735c760988310089dc074a73cc8e Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 9 Dec 2014 16:40:17 -0800 Subject: [PATCH 043/732] update error handling --- .classpath | 13 +- pom.xml | 5 + .../datastore/DatastoreServiceException.java | 77 ++++++---- .../gcloud/datastore/TransactionImpl.java | 4 +- .../datastore/DatastoreServiceTest.java | 132 +++++++++++++++++- 5 files changed, 193 insertions(+), 38 deletions(-) diff --git a/.classpath b/.classpath index 70f23326e0d2..d6cf6121af66 100644 --- a/.classpath +++ b/.classpath @@ -21,10 +21,17 @@ - - - + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 478c89fd38d0..d3431c1584fd 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,11 @@ joda-time RELEASE + + org.json + json + 20090211 + diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java index 94c16966f1fb..7e5ec384c65c 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java @@ -1,16 +1,17 @@ package com.google.gcloud.datastore; -import static com.google.common.base.MoreObjects.firstNonNull; - import com.google.api.services.datastore.client.DatastoreException; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableMap; -import java.util.HashMap; -import java.util.Map; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; public class DatastoreServiceException extends RuntimeException { private static final long serialVersionUID = 8170357898917041899L; - private static final Map HTTP_TO_CODE = new HashMap<>(); + private static final ImmutableMap REASON_TO_CODE; private final Code code; @@ -21,23 +22,22 @@ public class DatastoreServiceException extends RuntimeException { */ public enum Code { - ABORTED(409, true, "Request aborted"), - DEADLINE_EXCEEDED(403, true, "Deadline exceeded"), - UNAVAILABLE(503, true, "Could not reach service"), - FAILED_PRECONDITION(412, false, "Invalid request"), - INVALID_ARGUMENT(400, false, "Request parameter has an invalid value"), - PERMISSION_DENIED(403, false, "Unauthorized request"), - RESOURCE_EXHAUSTED(402, false, "Quota exceeded"), - INTERNAL(500, false, "Server returned an error"), - UNKNOWN(0, false, "Unknown failure"); + ABORTED(true, "Request aborted"), + DEADLINE_EXCEEDED(true, "Deadline exceeded"), + UNAVAILABLE(true, "Could not reach service"), + FAILED_PRECONDITION(false, "Invalid request"), + INVALID_ARGUMENT(false, "Request parameter has an invalid value"), + PERMISSION_DENIED(false, "Unauthorized request"), + RESOURCE_EXHAUSTED(false, "Quota exceeded"), + INTERNAL(false, "Server returned an error"), + UNKNOWN(false, "Unknown failure"); private final boolean isTransient; - private final String msg; + private final String defaultMessage; - Code(int httpStatus, boolean isTransient, String msg) { + Code(boolean isTransient, String msg) { this.isTransient = isTransient; - this.msg = msg; - HTTP_TO_CODE.put(httpStatus, this); + this.defaultMessage = msg; } /** @@ -48,16 +48,28 @@ public boolean isTransient() { return isTransient; } - DatastoreServiceException translate(DatastoreException exception) { - return new DatastoreServiceException(this, exception); + DatastoreServiceException translate(DatastoreException exception, String msg) { + return new DatastoreServiceException(this, msg, exception); } } - public DatastoreServiceException(Code code, Exception cause) { - super(code.msg, cause); + static { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Code code : Code.values()) { + builder.put(code.name(), code); + } + REASON_TO_CODE = builder.build(); + } + + public DatastoreServiceException(Code code, String msg, Exception cause) { + super(MoreObjects.firstNonNull(msg, code.defaultMessage), cause); this.code = code; } + public DatastoreServiceException(Code code, String msg) { + this(code, msg, null); + } + /** * Returns the code associated with this exception. */ @@ -72,9 +84,23 @@ public Code code() { * @throws DatastoreServiceException every time */ static DatastoreServiceException translateAndThrow(DatastoreException exception) { - throw firstNonNull(HTTP_TO_CODE.get(exception.getCode()), Code.UNKNOWN).translate(exception); + String message = exception.getMessage(); + String reason = ""; + if (message != null) { + try { + JSONObject json = new JSONObject(new JSONTokener(exception.getMessage())); + JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0); + reason = error.getString("reason"); + message = error.getString("message"); + } catch (JSONException ex) { + // ignore - will be converted to unknown + } + } + Code code = MoreObjects.firstNonNull(REASON_TO_CODE.get(reason), Code.UNKNOWN); + throw code.translate(exception, message); } + /** * Throw a DatastoreServiceException with {@code FAILED_PRECONDITION} code and the {@code msg} * in a nested exception. @@ -82,9 +108,6 @@ static DatastoreServiceException translateAndThrow(DatastoreException exception) * @throws DatastoreServiceException every time */ static DatastoreServiceException throwInvalidRequest(String msg, Object... params) { - if (params.length > 0) { - msg = String.format(msg, params); - } - throw new DatastoreServiceException(Code.FAILED_PRECONDITION, new RuntimeException(msg)); + throw new DatastoreServiceException(Code.FAILED_PRECONDITION, String.format(msg, params)); } } diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java index 17fe3e1339fc..0b6e4e81b24c 100644 --- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java +++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java @@ -55,7 +55,9 @@ public void commit() { @Override public void rollback() { super.checkValid(); - datastore.rollbackTransaction(transaction); + if (!wasRolledback) { + datastore.rollbackTransaction(transaction); + } wasRolledback = true; } diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java index 40397e35c47c..23567330bbc8 100644 --- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java +++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java @@ -29,6 +29,8 @@ public class DatastoreServiceTest { private static final Key KEY1 = PARTIAL_KEY1.newKey("name"); private static final Key KEY2 = Key.builder(KEY1, KIND2, 1).build(); private static final Key KEY3 = Key.builder(KEY2).name("bla").build(); + private static final Key KEY4 = KEY2.newKey("newName1"); + private static final Key KEY5 = KEY2.newKey("newName2"); private static final KeyValue KEY_VALUE = new KeyValue(KEY1); private static final ListValue LIST_VALUE1 = ListValue.builder() .addValue(NULL_VALUE) @@ -80,7 +82,7 @@ public void setUp() { .build(); datastore = DatastoreServiceFactory.getDefault(options); // Prepare data for testing - datastore.delete(KEY1, KEY2, KEY3); + datastore.delete(KEY1, KEY2, KEY3, KEY4, KEY5); datastore.add(ENTITY1, ENTITY2); } @@ -91,13 +93,119 @@ public void testGetOptions() { @Test public void testNewTransactionCommit() { - // TODO (also try list value with different types) - fail("Not yet implemented"); + Transaction transaction = datastore.newTransaction(); + transaction.add(ENTITY3); + Entity entity2 = Entity.builder(ENTITY2) + .clearProperties() + .setNullProperty("bla") + .build(); + transaction.update(entity2); + transaction.delete(KEY1); + transaction.commit(); + + Iterator iter = datastore.get(KEY1, KEY2, KEY3); + assertNull(iter.next()); + assertEquals(entity2, iter.next()); + assertEquals(ENTITY3, iter.next()); + assertFalse(iter.hasNext()); + + try { + transaction.commit(); + fail("Expecting a failure"); + } catch (DatastoreServiceException ex) { + // expected to fail + } + + try { + transaction.rollback(); + fail("Expecting a failure"); + } catch (DatastoreServiceException ex) { + // expected to fail + } + + verifyNotUsable(transaction); + } + + @Test + public void testTransactionWithRead() { + Transaction transaction = datastore.newTransaction(); + assertNull(transaction.get(KEY3)); + transaction.add(ENTITY3); + transaction.commit(); + assertEquals(ENTITY3, datastore.get(KEY3)); + + transaction = datastore.newTransaction(); + assertEquals(ENTITY3, transaction.get(KEY3)); + // update entity3 during the transaction + datastore.put(Entity.builder(ENTITY3).clearProperties().build()); + transaction.update(ENTITY2); + try { + transaction.commit(); + fail("Expecting a failure"); + } catch (DatastoreServiceException expected) { + expected.printStackTrace(); + assertEquals(DatastoreServiceException.Code.ABORTED, expected.code()); + } } @Test public void testNewTransactionRollback() { - fail("Not yet implemented"); + Transaction transaction = datastore.newTransaction(); + transaction.add(ENTITY3); + Entity entity2 = Entity.builder(ENTITY2) + .clearProperties() + .setNullProperty("bla") + .setListProperty("list3", new StringValue("bla"), StringValue.builder("bla").build()) + .build(); + transaction.update(entity2); + transaction.delete(KEY1); + transaction.rollback(); + transaction.rollback(); // should be safe to repeat rollback calls + + try { + transaction.commit(); + fail("Expecting a failure"); + } catch (DatastoreServiceException ex) { + // expected to fail + } + + verifyNotUsable(transaction); + + Iterator iter = datastore.get(KEY1, KEY2, KEY3); + assertEquals(ENTITY1, iter.next()); + assertEquals(ENTITY2, iter.next()); + assertNull(iter.next()); + assertFalse(iter.hasNext()); + } + + private void verifyNotUsable(DatastoreWriter writer) { + try { + writer.add(ENTITY3); + fail("Expecting a failure"); + } catch (DatastoreServiceException ex) { + // expected to fail + } + + try { + writer.put(ENTITY3); + fail("Expecting a failure"); + } catch (DatastoreServiceException ex) { + // expected to fail + } + + try { + writer.update(ENTITY3); + fail("Expecting a failure"); + } catch (DatastoreServiceException ex) { + // expected to fail + } + + try { + writer.delete(ENTITY3.key()); + fail("Expecting a failure"); + } catch (DatastoreServiceException ex) { + // expected to fail + } } @Test @@ -108,10 +216,10 @@ public void testNewBatchWriter() { .clearProperties() .setNullProperty("bla") .build(); - Entity entity4 = Entity.builder(KEY2.newKey("newName1")) + Entity entity4 = Entity.builder(KEY4) .setProperty("value", new StringValue("value")) .build(); - Entity entity5 = Entity.builder(KEY2.newKey("newName2")) + Entity entity5 = Entity.builder(KEY5) .setStringProperty("value", "value") .build(); batchWriter.add(entity4, entity5); @@ -127,9 +235,12 @@ public void testNewBatchWriter() { try { batchWriter.submit(); + fail("Expecting a failure"); } catch (DatastoreServiceException ex) { // expected to fail } + verifyNotUsable(batchWriter); + batchWriter = datastore.newBatchWriter(); batchWriter.delete(entity4.key(), entity5.key()); batchWriter.update(ENTITY1, ENTITY2, ENTITY3); @@ -142,7 +253,11 @@ public void testNewBatchWriter() { assertNull(entities.next()); assertFalse(entities.hasNext()); - // TODO need to cover more edge cases (ds failures, re-use of same entities,..) + // TODO need to cover the cases of: + // delete after put/add/update + // put after delete/add/update + // update after delete/add/put + // add after delete/update/put } @Test @@ -235,6 +350,7 @@ public void testGetArray() { assertFalse(entity3.hasProperty("bla")); try { entity3.stringProperty("str"); + fail("Expecting a failure"); } catch (DatastoreServiceException expected) { // expected - no such property } @@ -250,6 +366,7 @@ public void testAdd() { try { datastore.add(ENTITY1); + fail("Expecting a failure"); } catch (DatastoreServiceException expected) { // expected; } @@ -266,6 +383,7 @@ public void testUpdate() { try { datastore.update(ENTITY3); + fail("Expecting a failure"); } catch (DatastoreServiceException expected) { // expected; } From 5474d4f60576ced7a2584212a41e876e38366a1b Mon Sep 17 00:00:00 2001 From: aozarov Date: Tue, 9 Dec 2014 17:42:36 -0800 Subject: [PATCH 044/732] query impl - work in progress --- .../google/gcloud/datastore/QueryResult.java | 26 ++++++-- .../gcloud/datastore/QueryResultImpl.java | 64 +++++++++++++++++++ 2 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/google/gcloud/datastore/QueryResultImpl.java diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java index 446fc6dc43e4..8029c2654ce3 100644 --- a/src/main/java/com/google/gcloud/datastore/QueryResult.java +++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java @@ -1,5 +1,7 @@ package com.google.gcloud.datastore; +import com.google.api.services.datastore.DatastoreV1; + import java.util.Iterator; /** @@ -19,9 +21,23 @@ public interface QueryResult extends Iterator { */ enum Type { - FULL(Entity.class), - PROJECTION(PartialEntity.class), - KEY_ONLY(Key.class); + FULL(Entity.class) { + @Override Object convert(DatastoreV1.Entity value) { + return Entity.fromPb(value); + } + }, + + PROJECTION(PartialEntity.class) { + @Override Object convert(DatastoreV1.Entity value) { + return PartialEntity.fromPb(value); + } + }, + + KEY_ONLY(Key.class) { + @Override Object convert(DatastoreV1.Entity value) { + return Key.fromPb(value.getKey()); + } + }; private final Class resultClass; @@ -29,9 +45,11 @@ enum Type { this.resultClass = resultClass; } - Class getResultClass() { + public Class getResultClass() { return resultClass; } + + abstract Object convert(DatastoreV1.Entity value); } /** diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java new file mode 100644 index 000000000000..aa237944ca42 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java @@ -0,0 +1,64 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.api.services.datastore.DatastoreV1.EntityResult; +import com.google.common.collect.ImmutableMap; + +import java.util.Iterator; + +public class QueryResultImpl implements QueryResult { + + private static final ImmutableMap + RESULT_TYPE_CONVERTER; + + private final QueryResult.Type type; + private Iterator entityResultPbIter; + private boolean moreResult; + + static { + ImmutableMap.Builder builder = + ImmutableMap.builder(); + for (DatastoreV1.EntityResult.ResultType type : DatastoreV1.EntityResult.ResultType.values()) { + builder.put(type, QueryResult.Type.valueOf(type.name())); + } + RESULT_TYPE_CONVERTER = builder.build(); + } + + QueryResultImpl(DatastoreV1.Query DatastoreV1.QueryResultBatch resultBatch) { + type = RESULT_TYPE_CONVERTER.get(resultBatch.getEntityResultType()); + entityResultPbIter = resultBatch.getEntityResultList().iterator(); + moreResult = + DatastoreV1.QueryResultBatch.MoreResultsType.NOT_FINISHED == resultBatch.getMoreResults(); + } + + @Override + public boolean hasNext() { + if (entityResultPbIter.hasNext()) { + return true; + } else if (moreResult) { + // need to fetch more and update results (and more results) + } + return false; + } + + @Override + public V next() { + return (V) type.convert(entityResultPbIter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("QueryResult is read-only"); + } + + @Override + public QueryResult.Type getType() { + return type; + } + + @Override + public Cursor getCursor() { + // TODO(ozarov): implement when v1beta3 is available + return null; + } +} From 7630e70cd92900fa2958e4b123fce6edbb6a8ddf Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 10 Dec 2014 16:30:47 -0800 Subject: [PATCH 045/732] added Gql --- .../google/gcloud/datastore/BatchWriter.java | 5 + .../com/google/gcloud/datastore/Blob.java | 20 +- .../com/google/gcloud/datastore/Cursor.java | 58 +-- .../gcloud/datastore/DatastoreReader.java | 2 +- .../datastore/DatastoreServiceImpl.java | 26 +- .../com/google/gcloud/datastore/DateTime.java | 21 +- .../google/gcloud/datastore/EntityValue.java | 1 + .../com/google/gcloud/datastore/GqlQuery.java | 329 +++++++++++++++++- .../java/com/google/gcloud/datastore/Key.java | 3 + .../google/gcloud/datastore/ListValue.java | 11 +- .../gcloud/datastore/PartialEntity.java | 3 + .../google/gcloud/datastore/PartialKey.java | 3 + .../google/gcloud/datastore/PathElement.java | 3 + .../com/google/gcloud/datastore/Query.java | 40 ++- .../google/gcloud/datastore/QueryResult.java | 13 +- .../gcloud/datastore/QueryResultImpl.java | 51 ++- .../google/gcloud/datastore/Transaction.java | 9 +- .../gcloud/datastore/TransactionImpl.java | 2 +- .../gcloud/datastore/SerializationTest.java | 18 +- 19 files changed, 544 insertions(+), 74 deletions(-) diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java index 889ff425a1ab..22fe2ded6179 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java @@ -1,5 +1,10 @@ package com.google.gcloud.datastore; +/** + * An interface to represent a batch of write operations. + * Any write operation that is applied on a batch will only be sent + * to the Datastore upon {@link #submit} and with as few RPC calls as possible. + */ public interface BatchWriter extends DatastoreWriter { /** diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java index 4edfdabe7ebf..a176a10e25c3 100644 --- a/src/main/java/com/google/gcloud/datastore/Blob.java +++ b/src/main/java/com/google/gcloud/datastore/Blob.java @@ -3,9 +3,12 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.services.datastore.DatastoreV1; +import com.google.api.services.datastore.DatastoreV1.Value; import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; @@ -20,12 +23,12 @@ * * @see Google Cloud Datastore Entities, Properties, and Keys */ -public final class Blob implements java.io.Serializable { +public final class Blob extends Serializable { private static final long serialVersionUID = 3835421019618247721L; private static final int MAX_LENGTH = 1_000_000; - private final ByteString byteString; + private final transient ByteString byteString; Blob(ByteString byteString, boolean enforceLimits) { this.byteString = checkNotNull(byteString); @@ -54,6 +57,9 @@ public int hashCode() { @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (!(obj instanceof Blob)) { return false; } @@ -134,4 +140,14 @@ public static Blob copyFrom(InputStream input) throws IOException { } return copyFrom(bytes.toByteArray()); } + + @Override + protected Value toPb() { + return DatastoreV1.Value.newBuilder().setBlobValue(byteString).build(); + } + + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return new Blob(DatastoreV1.Value.parseFrom(bytesPb).getBlobValue(), false); + } } diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java index 5b3ba6981af3..e95f25144c12 100644 --- a/src/main/java/com/google/gcloud/datastore/Cursor.java +++ b/src/main/java/com/google/gcloud/datastore/Cursor.java @@ -3,83 +3,99 @@ import static com.google.common.base.Preconditions.checkNotNull; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.api.services.datastore.DatastoreV1; +import com.google.api.services.datastore.DatastoreV1.Value; import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; -import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; -import java.util.Arrays; /** * A Google Cloud Datastore cursor. * The cursor can be used to as a starting point or an ending point for a {@link Query} */ -public final class Cursor implements Serializable { +public final class Cursor extends Serializable { private static final long serialVersionUID = -1423744878777486541L; - private final byte[] bytes; + private final transient ByteString byteString; - public Cursor(byte[] bytes) { - this.bytes = checkNotNull(bytes); + Cursor(ByteString byteString) { + this.byteString = byteString; } @Override public int hashCode() { - return Arrays.hashCode(bytes); + return byteString.hashCode(); } @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (!(obj instanceof Cursor)) { return false; } - - return Arrays.equals(bytes, ((Cursor) obj).bytes); + return byteString.equals(((Cursor) obj).byteString); } @Override public String toString() { ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); StringBuilder stBuilder = new StringBuilder(); - for (byte b : bytes) { - stBuilder.append(String.format("%02x", b)); + for (int i = 0; i < byteString.size(); i++) { + stBuilder.append(String.format("%02x", byteString.byteAt(i))); } return toStringHelper.add("bytes", stBuilder.toString()).toString(); } + ByteString byteString() { + return byteString; + } + /** * Returns the cursor in an encoded form that can be used as part of a URL. */ public String toUrlSafe() { try { - return URLEncoder.encode(toString(), UTF_8.name()); + return URLEncoder.encode(toPb().toString(), UTF_8.name()); } catch (UnsupportedEncodingException e) { throw new RuntimeException("Unxpeced encoding exception", e); } } - ByteString toPb() { - return ByteString.copyFrom(bytes); - } - /** * Create a {@code Cursor} given its URL safe encoded form. */ public static Cursor fromUrlSafe(String urlSafe) { try { String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name()); - ByteString byteString = ByteString.copyFromUtf8(utf8Str); - return fromPb(byteString); - } catch (UnsupportedEncodingException e) { + return fromPb(DatastoreV1.Value.parseFrom(utf8Str.getBytes())); + } catch (UnsupportedEncodingException | InvalidProtocolBufferException e) { throw new RuntimeException("Unxpeced decoding exception", e); } } - private static Cursor fromPb(ByteString byteString) { - return new Cursor(byteString.toByteArray()); + public static Cursor copyFrom(byte[] bytes) { + return new Cursor(ByteString.copyFrom(checkNotNull(bytes))); + } + + @Override + protected Value toPb() { + return DatastoreV1.Value.newBuilder().setBlobValue(byteString).build(); + } + + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.Value.parseFrom(bytesPb)); + } + + static Cursor fromPb(DatastoreV1.Value valuePb) { + return new Cursor(valuePb.getBlobValue()); } } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java index ac2e048eff0f..ec7052e414e0 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java @@ -27,5 +27,5 @@ public interface DatastoreReader { * * @throws DatastoreServiceException upon failure. */ - QueryResult runQuery(Query query); + QueryResult runQuery(Query query); } diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 250fe0048360..891ed8fd7974 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -1,7 +1,6 @@ package com.google.gcloud.datastore; import com.google.api.services.datastore.DatastoreV1; -import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse; import com.google.api.services.datastore.client.Datastore; import com.google.api.services.datastore.client.DatastoreException; import com.google.common.collect.AbstractIterator; @@ -49,11 +48,27 @@ public Transaction newTransaction(TransactionOption... transactionOption) { } @Override - public QueryResult runQuery(Query query) { - // TODO To implement - throw new RuntimeException("Not implemented yet"); + public QueryResult runQuery(Query query) { + DatastoreV1.RunQueryRequest.Builder requestPbBuilder = DatastoreV1.RunQueryRequest.newBuilder(); + DatastoreV1.PartitionId.Builder partitionId = DatastoreV1.PartitionId.newBuilder(); + partitionId.setDatasetId(options.dataset()); + String namespace = query.namespace() != null ? query.namespace() : options.namespace(); + if (namespace != null) { + partitionId.setNamespace(namespace); + } + requestPbBuilder.setPartitionId(partitionId.build()); + query.populatePb(requestPbBuilder, 0, null); + DatastoreV1.RunQueryRequest requestPb = requestPbBuilder.build(); + return new QueryResultImpl<>(this, query, requestPb, runQuery(requestPb).getBatch()); } + DatastoreV1.RunQueryResponse runQuery(DatastoreV1.RunQueryRequest requestPb) { + try { + return datastore.runQuery(requestPb); + } catch (DatastoreException e) { + throw DatastoreServiceException.translateAndThrow(e); + } + } @Override public Key allocateId(PartialKey key) { @@ -211,7 +226,8 @@ void comitMutation(DatastoreV1.CommitRequest.Builder requestPb) { ByteString requestTransactionId(DatastoreV1.BeginTransactionRequest.Builder requestPb) { try { - BeginTransactionResponse responsePb = datastore.beginTransaction(requestPb.build()); + DatastoreV1.BeginTransactionResponse responsePb = + datastore.beginTransaction(requestPb.build()); return responsePb.getTransaction(); } catch (DatastoreException e) { throw DatastoreServiceException.translateAndThrow(e); diff --git a/src/main/java/com/google/gcloud/datastore/DateTime.java b/src/main/java/com/google/gcloud/datastore/DateTime.java index 7ade5c46d229..e04d603ff031 100644 --- a/src/main/java/com/google/gcloud/datastore/DateTime.java +++ b/src/main/java/com/google/gcloud/datastore/DateTime.java @@ -2,6 +2,10 @@ import static com.google.common.base.Preconditions.checkNotNull; +import com.google.api.services.datastore.DatastoreV1; +import com.google.api.services.datastore.DatastoreV1.Value; +import com.google.protobuf.InvalidProtocolBufferException; + import org.joda.time.format.ISODateTimeFormat; import java.util.Calendar; @@ -13,11 +17,11 @@ * * @see Google Cloud Datastore Entities, Properties, and Keys */ -public final class DateTime implements java.io.Serializable { +public final class DateTime extends Serializable { private static final long serialVersionUID = 7343324797621228378L; - private final long timestampMicroseconds; + private final transient long timestampMicroseconds; DateTime(long timestampMicroseconds) { this.timestampMicroseconds = timestampMicroseconds; @@ -35,6 +39,9 @@ public int hashCode() { @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (!(obj instanceof DateTime)) { return false; } @@ -70,4 +77,14 @@ public static DateTime copyFrom(Date date) { public static DateTime copyFrom(Calendar calendar) { return copyFrom(calendar.getTime()); } + + @Override + protected Value toPb() { + return DatastoreV1.Value.newBuilder().setIntegerValue(timestampMicroseconds).build(); + } + + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return new DateTime(DatastoreV1.Value.parseFrom(bytesPb).getIntegerValue()); + } } diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java index 3b89d81bb4be..243c1bac7be9 100644 --- a/src/main/java/com/google/gcloud/datastore/EntityValue.java +++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java @@ -42,6 +42,7 @@ private Builder() { @Override public Builder indexed(boolean indexed) { + // see b/8730533 Preconditions.checkArgument(!indexed, "EntityValue can't be indexed"); return super.indexed(indexed); } diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java index 2f8f87ba0f18..ad6e73f867ea 100644 --- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java +++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java @@ -1,26 +1,337 @@ package com.google.gcloud.datastore; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.api.services.datastore.DatastoreV1; +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Booleans; +import com.google.common.primitives.Doubles; +import com.google.common.primitives.Longs; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -public class GqlQuery extends Serializable implements Query { +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +/** + * A Google Cloud Datastore GQL. + * + * @see GQL Reference + */ +public final class GqlQuery extends Query { private static final long serialVersionUID = 5988280590929540569L; + private final transient String queryString; + private final transient boolean allowLiteral; + private final transient ImmutableList nameArgs; + private final transient ImmutableList numberArgs; + + private static final class Argument extends Serializable { + + private static final long serialVersionUID = 1976895435257636275L; + + private final transient String name; + private final transient Cursor cursor; + private final transient Value value; + + Argument(String name, Cursor cursor) { + this.name = name; + this.cursor = cursor; + value = null; + } + + Argument(String name, Value value) { + this.name = name; + this.value = value; + cursor = null; + } + + String name() { + return name; + } + + @Override + public int hashCode() { + return Objects.hash(name, cursor, value); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof Argument)) { + return false; + } + Argument other = (Argument) obj; + return Objects.equals(name, other.name) + && Objects.equals(cursor, other.cursor) + && Objects.equals(value, other.value); + } + + @Override + protected DatastoreV1.GqlQueryArg toPb() { + DatastoreV1.GqlQueryArg.Builder argPb = DatastoreV1.GqlQueryArg.newBuilder(); + if (name != null) { + argPb.setName(name); + } + if (cursor != null) { + argPb.setCursor(cursor.byteString()); + } + if (value != null) { + argPb.setValue(value.toPb()); + } + return argPb.build(); + } + + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(DatastoreV1.GqlQueryArg.parseFrom(bytesPb)); + } + + static Argument fromPb(DatastoreV1.GqlQueryArg argPb) { + String name = argPb.hasName() ? argPb.getName() : null; + if (argPb.hasCursor()) { + return new Argument(name, new Cursor(argPb.getCursor())); + } + return new Argument(name, Value.fromPb(argPb.getValue())); + } + } + + public static final class Builder { + + private String namespace; + private String queryString; + private boolean allowLiteral; + private Map nameArgs = new TreeMap<>(); + private List numberArgs = new LinkedList<>(); + + private Builder(String query) { + queryString = checkNotNull(query); + } + + public Builder query(String query) { + queryString = checkNotNull(query); + return this; + } + + public Builder namespace(String namespace) { + this.namespace = namespace; + return this; + } + + public Builder allowLiteral(boolean allowLiteral) { + this.allowLiteral = allowLiteral; + return this; + } + + public Builder clearArguments() { + nameArgs.clear(); + numberArgs.clear(); + return this; + } + + public Builder setArgument(String name, Cursor cursor) { + nameArgs.put(name, new Argument(name, cursor)); + return this; + } + + public Builder setArgument(String name, String... value) { + nameArgs.put(name, toArgument(name, StringValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder setArgument(String name, long... value) { + nameArgs.put(name, toArgument(name, LongValue.MARSHALLER, Longs.asList(value))); + return this; + } + + public Builder setArgument(String name, double... value) { + nameArgs.put(name, toArgument(name, DoubleValue.MARSHALLER, Doubles.asList(value))); + return this; + } + + public Builder setArgument(String name, boolean... value) { + nameArgs.put(name, toArgument(name, BooleanValue.MARSHALLER, Booleans.asList(value))); + return this; + } + + public Builder setArgument(String name, DateTime... value) { + nameArgs.put(name, toArgument(name, DateTimeValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder setArgument(String name, Key... value) { + nameArgs.put(name, toArgument(name, KeyValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder setArgument(String name, PartialEntity... value) { + nameArgs.put(name, toArgument(name, EntityValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder setArgument(String name, Blob... value) { + nameArgs.put(name, toArgument(name, BlobValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder addArgument(Cursor cursor) { + numberArgs.add(new Argument(null, cursor)); + return this; + } + + public Builder addArgument(String... value) { + numberArgs.add(toArgument(StringValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder addArgument(long... value) { + numberArgs.add(toArgument(LongValue.MARSHALLER, Longs.asList(value))); + return this; + } + + public Builder addArgument(double... value) { + numberArgs.add(toArgument(DoubleValue.MARSHALLER, Doubles.asList(value))); + return this; + } + + public Builder addArgument(boolean... value) { + numberArgs.add(toArgument(BooleanValue.MARSHALLER, Booleans.asList(value))); + return this; + } + + public Builder addArgument(DateTime... value) { + numberArgs.add(toArgument(DateTimeValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder addArgument(Key... value) { + numberArgs.add(toArgument(KeyValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder addArgument(PartialEntity... value) { + numberArgs.add(toArgument(EntityValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + public Builder addArgument(Blob... value) { + numberArgs.add(toArgument(BlobValue.MARSHALLER, Arrays.asList(value))); + return this; + } + + @SuppressWarnings("rawtypes") + private static Argument toArgument(Value.BuilderFactory builderFactory, List values) { + return toArgument(null, builderFactory, values); + } + + public GqlQuery build() { + return new GqlQuery(this); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static Argument toArgument(String name, Value.BuilderFactory builderFactory, + List values) { + List> list = new ArrayList<>(values.size()); + for (Object object : values) { + list.add(builderFactory.newBuilder(object).build()); + } + Value value; + if (list.isEmpty()) { + value = new NullValue(); + } else if (list.size() == 1) { + value = list.get(0); + } else { + value = new ListValue(list); + } + return new Argument(name, value); + } + } + + private GqlQuery(Builder builder) { + super(builder.namespace); + queryString = builder.queryString; + allowLiteral = builder.allowLiteral; + nameArgs = ImmutableList.copyOf(builder.nameArgs.values()); + numberArgs = ImmutableList.copyOf(builder.numberArgs); + } + + @Override + public int hashCode() { + return Objects.hash(namespace(), queryString, allowLiteral, nameArgs, numberArgs); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof GqlQuery)) { + return false; + } + GqlQuery other = (GqlQuery) obj; + return Objects.equals(namespace(), other.namespace()) + && Objects.equals(queryString, other.queryString) + && allowLiteral == other.allowLiteral + && Objects.equals(nameArgs, other.nameArgs) + && Objects.equals(numberArgs, other.numberArgs); + } + @Override protected DatastoreV1.GqlQuery toPb() { - // TODO Auto-generated method stub - return null; + DatastoreV1.GqlQuery.Builder queryPb = DatastoreV1.GqlQuery.newBuilder(); + queryPb.setQueryString(queryString); + queryPb.setAllowLiteral(allowLiteral); + for (Argument argument : nameArgs) { + queryPb.addNameArg(argument.toPb()); + } + for (Argument argument : numberArgs) { + queryPb.addNumberArg(argument.toPb()); + } + return queryPb.build(); } @Override - protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { - // TODO Auto-generated method stub - return null; + protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead, + ByteString batchCursor) { + if (batchCursor == null) { + requestPb.setGqlQuery(toPb()); + return; + } + // see b/18705483 + throw new UnsupportedOperationException("paging not implemented yet"); + } + + @Override + protected Object fromPb(String namespace, byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb)); + } + + static GqlQuery fromPb(String namespace, DatastoreV1.GqlQuery queryPb) { + Builder builder = new Builder(queryPb.getQueryString()); + builder.namespace(namespace); + if (queryPb.hasAllowLiteral()) { + builder.allowLiteral = queryPb.getAllowLiteral(); + } + for (DatastoreV1.GqlQueryArg nameArg : queryPb.getNameArgList()) { + Argument argument = Argument.fromPb(nameArg); + builder.nameArgs.put(argument.name(), argument); + } + for (DatastoreV1.GqlQueryArg numberArg : queryPb.getNumberArgList()) { + Argument argument = Argument.fromPb(numberArg); + builder.numberArgs.add(argument); + } + return builder.build(); } - static GqlQuery fromPb(DatastoreV1.GqlQuery queryPb) { - // TODO Auto-generated method stub - return null; + public static Builder builder(String query) { + return new Builder(query); } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index b8a027069a85..34ce97f8972d 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -148,6 +148,9 @@ public int hashCode() { @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (!(obj instanceof Key)) { return false; } diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java index cd33c96dcff0..b9d7aa66c21c 100644 --- a/src/main/java/com/google/gcloud/datastore/ListValue.java +++ b/src/main/java/com/google/gcloud/datastore/ListValue.java @@ -68,6 +68,7 @@ public Builder addValue(Value first, Value... other) { @Override public Builder indexed(boolean indexed) { + // see b/18704917 DatastoreServiceException.throwInvalidRequest("ListValue can't specify index"); return this; } @@ -78,10 +79,10 @@ public Builder indexed(boolean indexed) { * @see com.google.gcloud.datastore.Value.BaseBuilder#set(java.lang.Object) */ @Override - public Builder set(List> properties) { + public Builder set(List> values) { listBuilder = ImmutableList.>builder(); - for (Value property : properties) { - addValue(property); + for (Value value : values) { + addValue(value); } return this; } @@ -98,8 +99,8 @@ public ListValue build() { } } - public ListValue(List> properties) { - this(builder().set(properties)); + public ListValue(List> values) { + this(builder().set(values)); } public ListValue(Value first, Value... other) { diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java index ccd754bc0712..41cabfccb0ff 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java +++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java @@ -65,6 +65,9 @@ public int hashCode() { @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (!(obj instanceof PartialEntity)) { return false; } diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java index e470a71c3376..74f5d5157bf0 100644 --- a/src/main/java/com/google/gcloud/datastore/PartialKey.java +++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java @@ -52,6 +52,9 @@ public int hashCode() { @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (!(obj instanceof PartialKey) || !PartialKey.class.equals(obj.getClass())) { return false; } diff --git a/src/main/java/com/google/gcloud/datastore/PathElement.java b/src/main/java/com/google/gcloud/datastore/PathElement.java index 213c5a7bdae8..34dcc06de4e6 100644 --- a/src/main/java/com/google/gcloud/datastore/PathElement.java +++ b/src/main/java/com/google/gcloud/datastore/PathElement.java @@ -63,6 +63,9 @@ public int hashCode() { @Override public boolean equals(Object obj) { + if (obj == this) { + return true; + } if (!(obj instanceof PathElement)) { return false; } diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java index 822b7c6dc472..5479fa4f761e 100644 --- a/src/main/java/com/google/gcloud/datastore/Query.java +++ b/src/main/java/com/google/gcloud/datastore/Query.java @@ -1,7 +1,43 @@ package com.google.gcloud.datastore; -import java.io.Serializable; +import com.google.api.services.datastore.DatastoreV1; +import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.protobuf.ByteString; +import com.google.protobuf.GeneratedMessage; +import com.google.protobuf.InvalidProtocolBufferException; -public interface Query extends Serializable { +public abstract class Query extends Serializable { + + private static final long serialVersionUID = -2748141759901313101L; + + private final String namespace; + + Query(String namespace) { + this.namespace = namespace; + } + + public String namespace() { + return namespace; + } + + @Override + public String toString() { + ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); + toStringHelper.add("namespace", namespace); + toStringHelper.add("queryPb", super.toString()); + return toStringHelper.toString(); + } + + @Override + protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(namespace, bytesPb); + } + + protected abstract Object fromPb(String namespace, byte[] bytesPb) + throws InvalidProtocolBufferException; + + protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead, + ByteString batchCursor); } diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java index 8029c2654ce3..47466950205e 100644 --- a/src/main/java/com/google/gcloud/datastore/QueryResult.java +++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java @@ -11,7 +11,7 @@ * * @param V the type of values the result holds. */ -public interface QueryResult extends Iterator { +public interface QueryResult extends Iterator { /** * The result's type. @@ -22,19 +22,22 @@ public interface QueryResult extends Iterator { enum Type { FULL(Entity.class) { - @Override Object convert(DatastoreV1.Entity value) { + @SuppressWarnings("unchecked") + @Override Entity convert(DatastoreV1.Entity value) { return Entity.fromPb(value); } }, PROJECTION(PartialEntity.class) { - @Override Object convert(DatastoreV1.Entity value) { + @SuppressWarnings("unchecked") + @Override PartialEntity convert(DatastoreV1.Entity value) { return PartialEntity.fromPb(value); } }, KEY_ONLY(Key.class) { - @Override Object convert(DatastoreV1.Entity value) { + @SuppressWarnings("unchecked") + @Override Key convert(DatastoreV1.Entity value) { return Key.fromPb(value.getKey()); } }; @@ -49,7 +52,7 @@ public Class getResultClass() { return resultClass; } - abstract Object convert(DatastoreV1.Entity value); + abstract T convert(DatastoreV1.Entity value); } /** diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java index aa237944ca42..d7b48f20ade8 100644 --- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java +++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java @@ -1,19 +1,23 @@ package com.google.gcloud.datastore; import com.google.api.services.datastore.DatastoreV1; -import com.google.api.services.datastore.DatastoreV1.EntityResult; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.ByteString; import java.util.Iterator; -public class QueryResultImpl implements QueryResult { +class QueryResultImpl implements QueryResult { private static final ImmutableMap RESULT_TYPE_CONVERTER; + private final DatastoreServiceImpl datastore; + private final Query query; private final QueryResult.Type type; - private Iterator entityResultPbIter; - private boolean moreResult; + private DatastoreV1.RunQueryRequest requestPb; + private Iterator entityResultPbIter; + private ByteString endCursor; + private int count; static { ImmutableMap.Builder builder = @@ -24,26 +28,39 @@ public class QueryResultImpl implements QueryResult { RESULT_TYPE_CONVERTER = builder.build(); } - QueryResultImpl(DatastoreV1.Query DatastoreV1.QueryResultBatch resultBatch) { - type = RESULT_TYPE_CONVERTER.get(resultBatch.getEntityResultType()); - entityResultPbIter = resultBatch.getEntityResultList().iterator(); - moreResult = - DatastoreV1.QueryResultBatch.MoreResultsType.NOT_FINISHED == resultBatch.getMoreResults(); + QueryResultImpl(DatastoreServiceImpl datastore, Query query, + DatastoreV1.RunQueryRequest requestPb, DatastoreV1.QueryResultBatch resultPb) { + this.datastore = datastore; + this.query = query; + this.requestPb = requestPb; + type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType()); + } + + void setQueryResultBatch(DatastoreV1.QueryResultBatch resultPb) { + entityResultPbIter = resultPb.getEntityResultList().iterator(); + if (DatastoreV1.QueryResultBatch.MoreResultsType.NOT_FINISHED == resultPb.getMoreResults()) { + endCursor = resultPb.getEndCursor(); + } } @Override public boolean hasNext() { - if (entityResultPbIter.hasNext()) { - return true; - } else if (moreResult) { - // need to fetch more and update results (and more results) - } - return false; + return entityResultPbIter.hasNext() || endCursor != null; } @Override - public V next() { - return (V) type.convert(entityResultPbIter.next()); + public T next() { + if (!hasNext() && endCursor != null) { + DatastoreV1.RunQueryRequest.Builder requestPbBuilder = requestPb.toBuilder(); + query.populatePb(requestPbBuilder, count, endCursor); + DatastoreV1.RunQueryRequest newRequestPb = requestPbBuilder.build(); + DatastoreV1.RunQueryResponse responsePb = datastore.runQuery(newRequestPb); + requestPb = newRequestPb; + setQueryResultBatch(responsePb.getBatch()); + } + DatastoreV1.Entity entity = entityResultPbIter.next().getEntity(); + count++; + return type.convert(entity); } @Override diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java index 4138b8409d98..c04f81114db4 100644 --- a/src/main/java/com/google/gcloud/datastore/Transaction.java +++ b/src/main/java/com/google/gcloud/datastore/Transaction.java @@ -4,6 +4,13 @@ /** * A Google cloud datastore transaction. + * Any write operation that is applied on a transaction will only be sent + * to the Datastore upon {@link #commit}. A call to {@link #rollback} will invalidate + * the transaction and discard the changes. Any read operation that is done by a transaction + * will be part of it and therefore a {@code commit} is guaranteed to fail if an entity + * was modified outside of the transaction after it was read. Write operation on this + * transaction will not be reflected by read operation (as the changes are only sent to + * the Datastore upon {@code commit}. * * @see Google Cloud Datastore transactions */ @@ -41,7 +48,7 @@ public interface Transaction extends DatastoreReader, DatastoreWriter { * @throws DatastoreServiceException upon failure. */ @Override - QueryResult runQuery(Query query); + QueryResult runQuery(Query query); /** * Commit the transaction. diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java index 0b6e4e81b24c..78bb6f520355 100644 --- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java +++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java @@ -41,7 +41,7 @@ public Iterator get(Key key, Key... others) { } @Override - public QueryResult runQuery(Query query) { + public QueryResult runQuery(Query query) { checkValid(); // TODO To implement throw new RuntimeException("Not implemented yet"); diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java index 92ad4a672460..1bb592b4eb93 100644 --- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java +++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java @@ -1,5 +1,6 @@ package com.google.gcloud.datastore; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -23,6 +24,15 @@ public class SerializationTest { private static final PartialKey INCOMPLETE_KEY2 = PartialKey.builder(KEY1, "v").addAncestor("p", 1).build(); private static final Key KEY2 = Key.builder(KEY1, "v", 2).build(); + private static final DateTime DATE_TIME1 = DateTime.now(); + private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world")); + private static final Cursor CURSOR1 = Cursor.copyFrom(new byte[] {1,2}); + private static final GqlQuery GQL1 = + GqlQuery.builder("select * from kind1 where name = @name and age > @1") + .setArgument("name", "name1") + .addArgument(20) + .namespace("ns1") + .build(); private static final KeyValue KEY_VALUE = new KeyValue(KEY1); private static final NullValue NULL_VALUE = NullValue.builder().indexed(true).build(); private static final StringValue STRING_VALUE = new StringValue("hello"); @@ -31,8 +41,7 @@ public class SerializationTest { private static final BooleanValue BOOLEAN_VALUE = new BooleanValue(true); private static final DateTimeValue DATE_AND_TIME_VALUE = new DateTimeValue(DateTime.now()); - private static final BlobValue BLOB_VALUE = - new BlobValue(Blob.copyFrom(new byte[] {10, 0, -2})); + private static final BlobValue BLOB_VALUE = new BlobValue(BLOB1); private static final RawValue RAW_VALUE = new RawValue( DatastoreV1.Value.newBuilder().setBlobKeyValue("blob-key").setMeaning(18).build()); private static final Entity ENTITY1 = Entity.builder(KEY1).build(); @@ -42,6 +51,7 @@ public class SerializationTest { .setProperty("p1", StringValue.builder("hi1").meaning(10).build()) .setProperty("p2", StringValue.builder("hi2").meaning(11).indexed(false).build()) .setProperty("p3", LongValue.builder(100).indexed(false).meaning(100).build()) + .setBlobProperty("blob", BLOB1) .build(); private static final PartialEntity EMBEDDED_ENTITY1 = ENTITY1; private static final PartialEntity EMBEDDED_ENTITY2 = ENTITY2; @@ -94,8 +104,10 @@ public void testValues() throws Exception { @Test public void testTypes() throws Exception { Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2, - ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3}; + ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3, DATE_TIME1, + BLOB1, CURSOR1, GQL1}; for (Object obj : types) { + System.out.println("KOKO: " + obj); Object copy = serialiazeAndDeserialize(obj); assertEquals(obj, obj); assertEquals(obj, copy); From 61b8cc3c0cb0f6ce5c46252af401420fcea26f20 Mon Sep 17 00:00:00 2001 From: aozarov Date: Wed, 10 Dec 2014 16:31:12 -0800 Subject: [PATCH 046/732] work in progress for structured query --- .../gcloud/datastore/StructuredQuery.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/main/java/com/google/gcloud/datastore/StructuredQuery.java diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java new file mode 100644 index 000000000000..dcca9d7e45d2 --- /dev/null +++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java @@ -0,0 +1,65 @@ +package com.google.gcloud.datastore; + +import com.google.api.services.datastore.DatastoreV1; +import com.google.gcloud.datastore.DatastoreServiceException.Code; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + +public final class StructuredQuery extends Query { + + private static final long serialVersionUID = 546838955624019594L; + + + static class BaseBuilder { + // TODO: impelement and have sub-classes for keys-only, entity and projection + } + + private StructuredQuery(String namespace) { + super(namespace); + } + + @Override + public int hashCode() { + // implement + return 0; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + // implement + return false; + } + + @Override + protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead, + ByteString batchCursor) { + if (batchCursor == null) { + requestPb.setQuery(toPb()); + return; + } + throw new UnsupportedOperationException("paging not implemented yet"); + } + + @Override + protected Object fromPb(String namespace, byte[] bytesPb) throws InvalidProtocolBufferException { + return fromPb(namespace, DatastoreV1.Query.parseFrom(bytesPb)); + } + + @Override + protected DatastoreV1.Query toPb() { + // TODO Auto-generated method stub + return null; + } + + static StructuredQuery fromPb(String namespace, DatastoreV1.Query queryPb) { + return null; + } + + static DatastoreV1.Query nextQuery(DatastoreV1.Query query, ByteString cursor) { + // see b/18705483 + throw new DatastoreServiceException(Code.INTERNAL, "paging for gql results is not implemented"); + } +} From c0f18a3d6da46997b9d8c14dc37fd93937cab3e2 Mon Sep 17 00:00:00 2001 From: aozarov Date: Thu, 11 Dec 2014 14:29:33 -0800 Subject: [PATCH 047/732] some refactoring, update documentation and strcutured query work in progress --- .settings/org.eclipse.jdt.core.prefs | 87 ++++++++++++ .../google/gcloud/datastore/BaseEntity.java | 97 ++++++++------ .../google/gcloud/datastore/BatchWriter.java | 16 +++ .../gcloud/datastore/BatchWriterImpl.java | 25 ++-- .../com/google/gcloud/datastore/Blob.java | 2 +- .../datastore/DatastoreServiceImpl.java | 15 ++- .../com/google/gcloud/datastore/GqlQuery.java | 79 +++++------ .../java/com/google/gcloud/datastore/Key.java | 2 +- .../com/google/gcloud/datastore/Query.java | 88 +++++++++++- .../google/gcloud/datastore/QueryResult.java | 44 +++--- .../gcloud/datastore/QueryResultImpl.java | 3 + .../gcloud/datastore/StructuredQuery.java | 78 +++++++++-- .../google/gcloud/datastore/Transaction.java | 24 ++++ .../gcloud/datastore/TransactionImpl.java | 22 +-- .../com/google/gcloud/datastore/Value.java | 2 - .../google/gcloud/datastore/package-info.java | 22 +-- .../datastore/DatastoreServiceTest.java | 125 ++++++++---------- .../gcloud/datastore/SerializationTest.java | 28 ++-- 18 files changed, 515 insertions(+), 244 deletions(-) diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index fe5e054849a5..62f72418e536 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -2,7 +2,94 @@ <<<<<<<=HEAD >>>>>>>=ef72daa81c73f99e2156f5bfe8127591fc6358e9 eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=error +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=warning +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=warning +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=warning +org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=warning +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=warning +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=warning +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=warning +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=warning +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=error +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=warning +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=error +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=warning +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=warning +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=warning +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=disabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=warning +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=warning +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=warning +org.eclipse.jdt.core.compiler.problem.unusedParameter=warning +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=warning +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning org.eclipse.jdt.core.compiler.source=1.7 diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java index 41a6674cbcee..010c97dc256e 100644 --- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java +++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java @@ -11,7 +11,7 @@ import java.util.Set; /** - * A base class for entities. + * A base class for entities to hold the properties. */ abstract class BaseEntity extends Serializable { @@ -36,72 +36,78 @@ protected B self() { return (B) this; } - public B clearProperties() { + /** + * Clears all the properties. + */ + public B clear() { properties.clear(); return self(); } - public B removeProperty(String name) { + /** + * Removes a property with the given {@code name}. + */ + public B remove(String name) { properties.remove(name); return self(); } - public B setProperty(String name, Value value) { + public B set(String name, Value value) { properties.put(name, value); return self(); } - public B setNullProperty(String name) { + public B setNull(String name) { properties.put(name, new NullValue()); return self(); } - public B setStringProperty(String name, String value) { + public B set(String name, String value) { properties.put(name, new StringValue(value)); return self(); } - public B setLongProperty(String name, long value) { + public B set(String name, long value) { properties.put(name, new LongValue(value)); return self(); } - public B setDoubleProperty(String name, double value) { + public B set(String name, double value) { properties.put(name, new DoubleValue(value)); return self(); } - public B setBooleanProperty(String name, boolean value) { + public B set(String name, boolean value) { properties.put(name, new BooleanValue(value)); return self(); } - public B setDateTimeProperty(String name, DateTime value) { + public B set(String name, DateTime value) { properties.put(name, new DateTimeValue(value)); return self(); } - public B setKeyProperty(String name, Key value) { + public B set(String name, Key value) { properties.put(name, new KeyValue(value)); return self(); } - public B setEntityProperty(String name, PartialEntity value) { + public B set(String name, PartialEntity value) { properties.put(name, new EntityValue(value)); return self(); } - public B setListProperty(String name, List> values) { + public B set(String name, List> values) { properties.put(name, new ListValue(values)); return self(); } - public B setListProperty(String name, Value... value) { + public B set(String name, Value... value) { properties.put(name, new ListValue(Arrays.asList(value))); return self(); } - public B setBlobProperty(String name, Blob value) { + public B set(String name, Blob value) { properties.put(name, new BlobValue(value)); return self(); } @@ -118,18 +124,18 @@ protected BaseEntity(ImmutableSortedMap> properties) { } /** - * Returns {@code true} if there is such property with the given {@code name}. + * Returns {@code true} if the entity contains a property with the given {@code name}. */ - public boolean hasProperty(String name) { + public boolean contains(String name) { return properties.containsKey(name); } /** - * Returns the {@link Value} of property with the given {@code name}. + * Returns the {@link Value} for the given property {@code name}. * * @throws DatastoreServiceException if not such property. */ - public > V property(String name) { + public > V getValue(String name) { @SuppressWarnings("unchecked") V property = (V) properties.get(name); if (property == null) { @@ -138,63 +144,66 @@ public > V property(String name) { return property; } - public Type propertyType(String name) { - return property(name).type(); + public Type type(String name) { + return getValue(name).type(); } - public boolean isNullProperty(String name) { - return property(name) instanceof NullValue; + public boolean isNull(String name) { + return getValue(name) instanceof NullValue; } - public String stringProperty(String name) { - return ((StringValue) property(name)).get(); + public String getString(String name) { + return ((StringValue) getValue(name)).get(); } - public long longProperty(String name) { - return ((LongValue) property(name)).get(); + public long getLong(String name) { + return ((LongValue) getValue(name)).get(); } - public double doubleProperty(String name) { - return ((DoubleValue) property(name)).get(); + public double getDouble(String name) { + return ((DoubleValue) getValue(name)).get(); } - public boolean booleanProperty(String name) { - return ((BooleanValue) property(name)).get(); + public boolean getBoolean(String name) { + return ((BooleanValue) getValue(name)).get(); } - public DateTime dateTimeProperty(String name) { - return ((DateTimeValue) property(name)).get(); + public DateTime getDateTime(String name) { + return ((DateTimeValue) getValue(name)).get(); } - public Key keyProperty(String name) { - return ((KeyValue) property(name)).get(); + public Key getKey(String name) { + return ((KeyValue) getValue(name)).get(); } @SuppressWarnings("unchecked") - public T entityProperty(String name) { - return (T) ((EntityValue) property(name)).get(); + public T getEntity(String name) { + return (T) ((EntityValue) getValue(name)).get(); } - public List> listProperty(String name) { - return ((ListValue) property(name)).get(); + public List> getList(String name) { + return ((ListValue) getValue(name)).get(); } - public Blob blobProperty(String name) { - return ((BlobValue) property(name)).get(); + public Blob getBlob(String name) { + return ((BlobValue) getValue(name)).get(); } /** * Returns the property's value as a {@link RawValue}. */ - public RawValue asRawValueProperty(String name) { - Value value = property(name); + public RawValue asRawValue(String name) { + Value value = getValue(name); if (value instanceof RawValue) { return (RawValue) value; } return new RawValue(value.toPb()); } - public Set propertyNames() { + /** + * Returns the properties name. + */ + public Set names() { return properties.keySet(); } diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java index 22fe2ded6179..99aeafabd3d4 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java @@ -4,6 +4,17 @@ * An interface to represent a batch of write operations. * Any write operation that is applied on a batch will only be sent * to the Datastore upon {@link #submit} and with as few RPC calls as possible. + * A usage example: + * 
 {@code
+ *   Entity entity1 = datastore.get(key1);
+ *   BatchWriter batchWriter = datastore.newBatchWriter();
+ *   Entity entity2 = Entity.builder(key2).set("name", "John").build();
+ *   entity1 = Entity.builder(entity1).clear().setNull("bla").build();
+ *   Entity entity3 = Entity.builder(key3).set("title", new StringValue("title")).build();
+ *   batchWriter.update(entity1);
+ *   batchWriter.add(entity2, entity3);
+ *   batchWriter.submit();
+ * }
*/ public interface BatchWriter extends DatastoreWriter { @@ -27,4 +38,9 @@ public interface BatchWriter extends DatastoreWriter { * @throws DatastoreServiceException if there was any failure. */ void submit(); + + /** + * Returns {@code true} if batch is still active (was not submitted). + */ + boolean active(); } diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java index 89fbc60742e2..0564d9b18c30 100644 --- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java +++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java @@ -18,7 +18,7 @@ class BatchWriterImpl implements BatchWriter { private final boolean force; protected final DatastoreServiceImpl datastore; - private boolean wasSubmitted = false; + private boolean active = true; BatchWriterImpl(DatastoreServiceImpl datastore, BatchWriteOption... options) { this.datastore = datastore; @@ -31,9 +31,9 @@ class BatchWriterImpl implements BatchWriter { } } - protected void checkValid() { - if (wasSubmitted) { - throwInvalidRequest(getName() + " was already submitted"); + protected void checkActive() { + if (!active) { + throwInvalidRequest(getName() + " is no longer active"); } } @@ -43,7 +43,7 @@ protected String getName() { @Override public void add(Entity... entities) { - checkValid(); + checkActive(); for (Entity entity : entities) { Key key = entity.key(); if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) { @@ -60,7 +60,7 @@ public void add(Entity... entities) { @Override public void update(Entity... entities) { - checkValid(); + checkActive(); for (Entity entity : entities) { Key key = entity.key(); if (toDelete.contains(key)) { @@ -77,7 +77,7 @@ public void update(Entity... entities) { @Override public void put(Entity... entities) { - checkValid(); + checkActive(); for (Entity entity : entities) { Key key = entity.key(); toAdd.remove(key); @@ -89,7 +89,7 @@ public void put(Entity... entities) { @Override public void delete(Key... keys) { - checkValid(); + checkActive(); for (Key key : keys) { toAdd.remove(key); toUpdate.remove(key); @@ -100,7 +100,7 @@ public void delete(Key... keys) { @Override public void submit() { - checkValid(); + checkActive(); DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder(); for (Entity entity : toAdd.values()) { mutationPb.addInsert(entity.toPb()); @@ -120,7 +120,12 @@ public void submit() { DatastoreV1.CommitRequest.Builder requestPb = newCommitRequest(); requestPb.setMutation(mutationPb); datastore.comitMutation(requestPb); - wasSubmitted = true; + active = false; + } + + @Override + public boolean active() { + return active; } protected DatastoreV1.CommitRequest.Builder newCommitRequest() { diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java index a176a10e25c3..f1fabe2f4899 100644 --- a/src/main/java/com/google/gcloud/datastore/Blob.java +++ b/src/main/java/com/google/gcloud/datastore/Blob.java @@ -26,7 +26,7 @@ public final class Blob extends Serializable { private static final long serialVersionUID = 3835421019618247721L; - private static final int MAX_LENGTH = 1_000_000; + public static final int MAX_LENGTH = 1_000_000; private final transient ByteString byteString; diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java index 891ed8fd7974..8fe92407260f 100644 --- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java +++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java @@ -49,14 +49,21 @@ public Transaction newTransaction(TransactionOption... transactionOption) { @Override public QueryResult runQuery(Query query) { + return runQuery(null, query); + } + + QueryResult runQuery(DatastoreV1.ReadOptions readOptionsPb, Query query) { DatastoreV1.RunQueryRequest.Builder requestPbBuilder = DatastoreV1.RunQueryRequest.newBuilder(); - DatastoreV1.PartitionId.Builder partitionId = DatastoreV1.PartitionId.newBuilder(); - partitionId.setDatasetId(options.dataset()); + if (readOptionsPb != null) { + requestPbBuilder.setReadOptions(readOptionsPb); + } + DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder(); + partitionIdPb.setDatasetId(options.dataset()); String namespace = query.namespace() != null ? query.namespace() : options.namespace(); if (namespace != null) { - partitionId.setNamespace(namespace); + partitionIdPb.setNamespace(namespace); } - requestPbBuilder.setPartitionId(partitionId.build()); + requestPbBuilder.setPartitionId(partitionIdPb.build()); query.populatePb(requestPbBuilder, 0, null); DatastoreV1.RunQueryRequest requestPb = requestPbBuilder.build(); return new QueryResultImpl<>(this, query, requestPb, runQuery(requestPb).getBatch()); diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java index ad6e73f867ea..5e0a5c89323f 100644 --- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java +++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java @@ -23,7 +23,7 @@ * * @see GQL Reference */ -public final class GqlQuery extends Query { +final class GqlQuery extends Query { private static final long serialVersionUID = 5988280590929540569L; @@ -104,125 +104,130 @@ static Argument fromPb(DatastoreV1.GqlQueryArg argPb) { } } - public static final class Builder { + /** + * A GQL query builder. + */ + public static final class Builder { + private final ResultType resultType; private String namespace; private String queryString; private boolean allowLiteral; private Map nameArgs = new TreeMap<>(); private List numberArgs = new LinkedList<>(); - private Builder(String query) { + Builder(ResultType resultType, String query) { + this.resultType = resultType; queryString = checkNotNull(query); } - public Builder query(String query) { + public Builder query(String query) { queryString = checkNotNull(query); return this; } - public Builder namespace(String namespace) { + public Builder namespace(String namespace) { this.namespace = namespace; return this; } - public Builder allowLiteral(boolean allowLiteral) { + public Builder allowLiteral(boolean allowLiteral) { this.allowLiteral = allowLiteral; return this; } - public Builder clearArguments() { + public Builder clearArguments() { nameArgs.clear(); numberArgs.clear(); return this; } - public Builder setArgument(String name, Cursor cursor) { + public Builder setArgument(String name, Cursor cursor) { nameArgs.put(name, new Argument(name, cursor)); return this; } - public Builder setArgument(String name, String... value) { + public Builder setArgument(String name, String... value) { nameArgs.put(name, toArgument(name, StringValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder setArgument(String name, long... value) { + public Builder setArgument(String name, long... value) { nameArgs.put(name, toArgument(name, LongValue.MARSHALLER, Longs.asList(value))); return this; } - public Builder setArgument(String name, double... value) { + public Builder setArgument(String name, double... value) { nameArgs.put(name, toArgument(name, DoubleValue.MARSHALLER, Doubles.asList(value))); return this; } - public Builder setArgument(String name, boolean... value) { + public Builder setArgument(String name, boolean... value) { nameArgs.put(name, toArgument(name, BooleanValue.MARSHALLER, Booleans.asList(value))); return this; } - public Builder setArgument(String name, DateTime... value) { + public Builder setArgument(String name, DateTime... value) { nameArgs.put(name, toArgument(name, DateTimeValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder setArgument(String name, Key... value) { + public Builder setArgument(String name, Key... value) { nameArgs.put(name, toArgument(name, KeyValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder setArgument(String name, PartialEntity... value) { + public Builder setArgument(String name, PartialEntity... value) { nameArgs.put(name, toArgument(name, EntityValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder setArgument(String name, Blob... value) { + public Builder setArgument(String name, Blob... value) { nameArgs.put(name, toArgument(name, BlobValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder addArgument(Cursor cursor) { + public Builder addArgument(Cursor cursor) { numberArgs.add(new Argument(null, cursor)); return this; } - public Builder addArgument(String... value) { + public Builder addArgument(String... value) { numberArgs.add(toArgument(StringValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder addArgument(long... value) { + public Builder addArgument(long... value) { numberArgs.add(toArgument(LongValue.MARSHALLER, Longs.asList(value))); return this; } - public Builder addArgument(double... value) { + public Builder addArgument(double... value) { numberArgs.add(toArgument(DoubleValue.MARSHALLER, Doubles.asList(value))); return this; } - public Builder addArgument(boolean... value) { + public Builder addArgument(boolean... value) { numberArgs.add(toArgument(BooleanValue.MARSHALLER, Booleans.asList(value))); return this; } - public Builder addArgument(DateTime... value) { + public Builder addArgument(DateTime... value) { numberArgs.add(toArgument(DateTimeValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder addArgument(Key... value) { + public Builder addArgument(Key... value) { numberArgs.add(toArgument(KeyValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder addArgument(PartialEntity... value) { + public Builder addArgument(PartialEntity... value) { numberArgs.add(toArgument(EntityValue.MARSHALLER, Arrays.asList(value))); return this; } - public Builder addArgument(Blob... value) { + public Builder addArgument(Blob... value) { numberArgs.add(toArgument(BlobValue.MARSHALLER, Arrays.asList(value))); return this; } @@ -232,8 +237,8 @@ private static Argument toArgument(Value.BuilderFactory builderFactory, List return toArgument(null, builderFactory, values); } - public GqlQuery build() { - return new GqlQuery(this); + public GqlQuery build() { + return new GqlQuery<>(this); } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -255,8 +260,8 @@ private static Argument toArgument(String name, Value.BuilderFactory builderFact } } - private GqlQuery(Builder builder) { - super(builder.namespace); + private GqlQuery(Builder builder) { + super(builder.resultType, builder.namespace); queryString = builder.queryString; allowLiteral = builder.allowLiteral; nameArgs = ImmutableList.copyOf(builder.nameArgs.values()); @@ -276,7 +281,7 @@ public boolean equals(Object obj) { if (!(obj instanceof GqlQuery)) { return false; } - GqlQuery other = (GqlQuery) obj; + GqlQuery other = (GqlQuery) obj; return Objects.equals(namespace(), other.namespace()) && Objects.equals(queryString, other.queryString) && allowLiteral == other.allowLiteral @@ -310,12 +315,14 @@ protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int tot } @Override - protected Object fromPb(String namespace, byte[] bytesPb) throws InvalidProtocolBufferException { - return fromPb(namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb)); + protected Object fromPb(ResultType resultType, String namespace, byte[] bytesPb) + throws InvalidProtocolBufferException { + return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb)); } - static GqlQuery fromPb(String namespace, DatastoreV1.GqlQuery queryPb) { - Builder builder = new Builder(queryPb.getQueryString()); + static GqlQuery fromPb(ResultType resultType, String namespace, + DatastoreV1.GqlQuery queryPb) { + Builder builder = new Builder<>(resultType, queryPb.getQueryString()); builder.namespace(namespace); if (queryPb.hasAllowLiteral()) { builder.allowLiteral = queryPb.getAllowLiteral(); @@ -330,8 +337,4 @@ static GqlQuery fromPb(String namespace, DatastoreV1.GqlQuery queryPb) { } return builder.build(); } - - public static Builder builder(String query) { - return new Builder(query); - } } diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java index 34ce97f8972d..9687ea9e830e 100644 --- a/src/main/java/com/google/gcloud/datastore/Key.java +++ b/src/main/java/com/google/gcloud/datastore/Key.java @@ -107,7 +107,7 @@ public String name() { } /** - * Returns the key's id (as {@link #Long}) or name (as {@link String}). + * Returns the key's id (as {@link Long}) or name (as {@link String}). */ public Object nameOrId() { return leaf.nameOrId(); diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java index 5479fa4f761e..d0bcaf70de79 100644 --- a/src/main/java/com/google/gcloud/datastore/Query.java +++ b/src/main/java/com/google/gcloud/datastore/Query.java @@ -3,21 +3,78 @@ import com.google.api.services.datastore.DatastoreV1; import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.gcloud.datastore.QueryResult.Type; import com.google.protobuf.ByteString; import com.google.protobuf.GeneratedMessage; import com.google.protobuf.InvalidProtocolBufferException; +// TODO(ozarov): add a usage examples (gql and regular) +/** + * A Google Cloud Datastore query. + * + * @param the type of the values returned by this query. + * @see Datastore Queries + */ public abstract class Query extends Serializable { private static final long serialVersionUID = -2748141759901313101L; + private final ResultType resultType; private final String namespace; - Query(String namespace) { + public static class ResultType implements java.io.Serializable { + + private static final long serialVersionUID = 2104157695425806623L; + private static final ResultType UNKNOWN = new ResultType<>(null, null); + private static final ResultType FULL = new ResultType<>(Entity.class, Type.FULL); + private static final ResultType KEY_ONLY = new ResultType<>(Key.class, Type.KEY_ONLY); + private static final ResultType PROJECTION = + new ResultType<>(PartialEntity.class, Type.PROJECTION); + + + private final Class clazz; + private final Type type; + + private ResultType(Class clazz, Type type) { + this.clazz = clazz; + this.type = type; + } + + public Type getType() { + return type; + } + + public Class getResultClass() { + return clazz; + } + + public static ResultType unknown() { + return UNKNOWN; + } + + public static ResultType full() { + return FULL; + } + + public static ResultType projection() { + return PROJECTION; + } + + public static ResultType keyOnly() { + return KEY_ONLY; + } + } + + Query(ResultType resultType, String namespace) { + this.resultType = resultType; this.namespace = namespace; } + public ResultType resultType() { + return resultType; + } + public String namespace() { return namespace; } @@ -32,12 +89,37 @@ public String toString() { @Override protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException { - return fromPb(namespace, bytesPb); + return fromPb(resultType, namespace, bytesPb); } - protected abstract Object fromPb(String namespace, byte[] bytesPb) + protected abstract Object fromPb(ResultType resultType, String namespace, byte[] bytesPb) throws InvalidProtocolBufferException; protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead, ByteString batchCursor); + + /** + * Returns a new GQL query builder. + * + * @see GQL Reference + */ + public static GqlQuery.Builder builder(String gql) { + return builder(ResultType.unknown(), gql); + } + + /** + * Returns a new GQL query builder. + * + * @see GQL Reference + */ + public static GqlQuery.Builder builder(ResultType resultType, String gql) { + return new GqlQuery.Builder<>(resultType, gql); + } + + /** + * Returns a new structured query builder. + */ + public static StructuredQuery.FullBuilder builder() { + return new StructuredQuery.FullBuilder(); + } } diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java index 47466950205e..9045764077e4 100644 --- a/src/main/java/com/google/gcloud/datastore/QueryResult.java +++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java @@ -6,57 +6,53 @@ /** * The result of a Google Cloud Datastore query submission. - * Typically the result's type would be casted to its expected type (The {@link #getResultType()} - * method could be used when the type is not known). + * When the result is not typed it is possible to cast it to its appropriate type according to + * the {@link #getType} value. * * @param V the type of values the result holds. */ -public interface QueryResult extends Iterator { +public interface QueryResult extends Iterator { /** - * The result's type. - * FULL: A complete {@link Entity}. - * PROJECTION: A partial entity, represented by {@link PartialEntity}. - * KEY_ONLY: An entity's {@link Key}. + * Possible results types are: + * FULL: A complete {@link Entity}. + * PROJECTION: A partial entity, represented by {@link PartialEntity}. + * KEY_ONLY: An entity's {@link Key}. */ enum Type { - FULL(Entity.class) { + FULL { + @Override @SuppressWarnings("unchecked") - @Override Entity convert(DatastoreV1.Entity value) { + Entity convert(DatastoreV1.Entity value) { return Entity.fromPb(value); } }, - PROJECTION(PartialEntity.class) { + PROJECTION { + + @Override @SuppressWarnings("unchecked") - @Override PartialEntity convert(DatastoreV1.Entity value) { + PartialEntity convert(DatastoreV1.Entity value) { return PartialEntity.fromPb(value); } }, - KEY_ONLY(Key.class) { + KEY_ONLY { + + @Override @SuppressWarnings("unchecked") - @Override Key convert(DatastoreV1.Entity value) { + Key convert(DatastoreV1.Entity value) { return Key.fromPb(value.getKey()); } }; - private final Class resultClass; - - Type(Class resultClass) { - this.resultClass = resultClass; - } - - public Class getResultClass() { - return resultClass; - } - abstract T convert(DatastoreV1.Entity value); } /** - * This method can be used to verify the result type and to cast its value type accordingly. + * Returns the actual type of the result's values. + * When needed the result could be casted accordingly: * 
 {@code
    * Type.FULL -> (QueryResult)
    * Type.PROJECTION -> (QueryResult)
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index d7b48f20ade8..f24130212ab8 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -1,5 +1,6 @@
 package com.google.gcloud.datastore;
 
+import com.google.api.client.repackaged.com.google.common.base.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableMap;
 import com.google.protobuf.ByteString;
@@ -34,6 +35,8 @@ class QueryResultImpl implements QueryResult {
     this.query = query;
     this.requestPb = requestPb;
     type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
+    Preconditions.checkState(query.resultType().getType() == null
+        || query.resultType().getType() == type, "Unexpected result type");
   }
 
   void setQueryResultBatch(DatastoreV1.QueryResultBatch resultPb) {
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index dcca9d7e45d2..890dca19de4b 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -1,21 +1,74 @@
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.gcloud.datastore.DatastoreServiceException.Code;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 
-public final class StructuredQuery extends Query {
+final class StructuredQuery extends Query {
 
   private static final long serialVersionUID = 546838955624019594L;
 
 
-  static class BaseBuilder {
-    // TODO: impelement and have sub-classes for keys-only, entity and projection
+  static class BaseBuilder> {
+
+    private String kind;
+    private String namespace;
+    private Cursor startCursor;
+    private Cursor endCursor;
+    private Integer offset;
+    private Integer limit;
+
+
+    protected B self() {
+      return (B) this;
+    }
+
+    public B kind(String kind) {
+      this.kind = kind;
+      return self();
+    }
+
+    public B namespace(String namespace) {
+      this.namespace = namespace;
+      return self();
+    }
+
+    public B startCursor(Cursor startCursor) {
+      this.startCursor = startCursor;
+      return self();
+    }
+
+    public B encCursor(Cursor endCursor) {
+      this.endCursor = endCursor;
+      return self();
+    }
+
+    public B offset(int offset) {
+      this.offset = offset;
+      return self();
+    }
+
+    public B limit(int limit) {
+      this.limit = limit;
+      return self();
+    }
   }
 
-  private StructuredQuery(String namespace) {
-    super(namespace);
+  public static final class FullBuilder extends BaseBuilder {
+
+  }
+
+  public static final class KeyOnlyBuilder extends BaseBuilder {
+
+  }
+
+  public static final class ProjectionBuilder
+      extends BaseBuilder {
+
+  }
+
+  private StructuredQuery(ResultType resultType, String namespace) {
+    super(resultType, namespace);
   }
 
   @Override
@@ -44,8 +97,9 @@ protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int tot
   }
 
   @Override
-  protected Object fromPb(String namespace, byte[] bytesPb) throws InvalidProtocolBufferException {
-    return fromPb(namespace, DatastoreV1.Query.parseFrom(bytesPb));
+  protected Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
+      throws InvalidProtocolBufferException {
+    return fromPb(resultType, namespace, DatastoreV1.Query.parseFrom(bytesPb));
   }
 
   @Override
@@ -54,12 +108,8 @@ protected DatastoreV1.Query toPb() {
     return null;
   }
 
-  static StructuredQuery fromPb(String namespace, DatastoreV1.Query queryPb) {
+  static  StructuredQuery fromPb(ResultType resultType, String namespace,
+      DatastoreV1.Query queryPb) {
     return null;
   }
-
-  static DatastoreV1.Query nextQuery(DatastoreV1.Query query, ByteString cursor) {
-    // see b/18705483
-    throw new DatastoreServiceException(Code.INTERNAL, "paging for gql results is not implemented");
-  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index c04f81114db4..d47a63ddbf1e 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -11,8 +11,27 @@
  * was modified outside of the transaction after it was read. Write operation on this
  * transaction will not be reflected by read operation (as the changes are only sent to
  * the Datastore upon {@code commit}.
+ * A usage example:
+ * 
 {@code
+ *   Transaction transaction = datastore.newTransaction();
+ *   try {
+ *     Entity entity = transaction.get(key);
+ *     if (!entity.contains("last_name") || entity.isNull("last_name")) {
+ *       String[] name = entity.getString("name").split(" ");
+ *       entity = Entity.builder(entity).remove("name").set("first_name", name[0])
+ *           .set("last_name", name[1]).build();
+ *       transaction.update(entity);
+ *       transaction.commit();
+ *     }
+ *   } finally {
+ *     if (transaction.active()) {
+ *       transaction.rollback();
+ *     }
+ *   }
+ * } 

  *
  * @see Google Cloud Datastore transactions
+ *
  */
 public interface Transaction extends DatastoreReader, DatastoreWriter {
 
@@ -61,4 +80,9 @@ public interface Transaction extends DatastoreReader, DatastoreWriter {
    * Rollback the transaction.
    */
   void rollback();
+
+  /**
+   * Returns {@code true} if the transaction is still active (was not committed or rolledback).
+   */
+  boolean active();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 78bb6f520355..ee0dc31685ed 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -34,7 +34,7 @@ public Entity get(Key key) {
 
   @Override
   public Iterator get(Key key, Key... others) {
-    checkValid();
+    checkActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
     return datastore.get(readOptionsPb.build(), key, others);
@@ -42,9 +42,10 @@ public Iterator get(Key key, Key... others) {
 
   @Override
   public  QueryResult runQuery(Query query) {
-    checkValid();
-    // TODO To implement
-    throw new RuntimeException("Not implemented yet");
+    checkActive();
+    DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
+    readOptionsPb.setTransaction(transaction);
+    return datastore.runQuery(readOptionsPb.build(), query);
   }
 
   @Override
@@ -54,23 +55,28 @@ public void commit() {
 
   @Override
   public void rollback() {
-    super.checkValid();
+    super.checkActive();
     if (!wasRolledback) {
       datastore.rollbackTransaction(transaction);
     }
     wasRolledback = true;
   }
 
+  @Override
+  public boolean active() {
+    return super.active() && !wasRolledback;
+  }
+
   @Override
   protected String getName() {
     return "transaction";
   }
 
   @Override
-  protected void checkValid() {
-    super.checkValid();
+  protected void checkActive() {
+    super.checkActive();
     if (wasRolledback) {
-      throwInvalidRequest(getName() + " was already rolledback");
+      throwInvalidRequest(getName() + " is not active (was rolledback)");
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index 448efe37ed31..9c569ae24de4 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -19,8 +19,6 @@
  * Unsupported value (deprecated or unrecognized) would be represented by {@link RawValue}.
  *
  * @param  the type of the content for this value
- * @param 
 the type of this value
- * @param  the type of this value's builder
  */
 public abstract class Value extends Serializable {
 
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index 5b2b8b8e8db5..b186edb83b0e 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -1,6 +1,6 @@
 /**
  * A client to the Google Cloud Datastore.
- * Typical usage would be:
+ * A simple usage example:
  * 
 {@code
  * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
@@ -9,21 +9,21 @@
  * Entity entity = datastore.get(key);
  * if (entity == null) {
  *   entity = Entity.builder(key)
- *       .setStringProperty("name", "John Do")
- *       .setProperty("age", LongValue.builder(100).indexed(false).build())
- *       .setBooleanProperty("updated", false)
+ *       .set("name", "John Do")
+ *       .set("age", LongValue.builder(100).indexed(false).build())
+ *       .set("updated", false)
  *       .build();
  *   datastore.put(entity);
  * } else {
- *   boolean updated = entity.booleanProperty("updated");
+ *   boolean updated = entity.getBoolean("updated");
  *   if (!updated) {
- *     String[] name = entity.stringProperty("name").split(" ");
+ *     String[] name = entity.getString("name").split(" ");
  *     entity = Entity.builder(entity)
- *         .setStringProperty("name", name[0])
- *         .setProperty("last_name", StringValue.builder(name[1]).indexed(false).build())
- *         .setBooleanProperty("updated", true)
- *         .removeProperty("old_property")
- *         .setDoubleProperty("new_property", 1.1)
+ *         .set("name", name[0])
+ *         .set("last_name", StringValue.builder(name[1]).indexed(false).build())
+ *         .set("updated", true)
+ *         .remove("old_property")
+ *         .set("new_property", 1.1)
  *         .build();
  *     datastore.update(entity);
  *   }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 23567330bbc8..0e22bd8ce195 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -38,33 +38,16 @@ public class DatastoreServiceTest {
       .build();
   private static final ListValue LIST_VALUE2 = new ListValue(Collections.singletonList(KEY_VALUE));
   private static final PartialEntity PARTIAL_ENTITY1 = PartialEntity.builder(PARTIAL_KEY2)
-      .setProperty("str", STR_VALUE)
-      .setProperty("bool", BOOL_VALUE)
-      .setProperty("list", LIST_VALUE1)
-      .build();
+      .set("str", STR_VALUE).set("bool", BOOL_VALUE).set("list", LIST_VALUE1).build();
   private static final PartialEntity PARTIAL_ENTITY2 = PartialEntity.builder(PARTIAL_ENTITY1)
-      .removeProperty("str")
-      .setBooleanProperty("bool", true)
-      .setListProperty("list", LIST_VALUE1.get())
-      .build();
-  private static final Entity ENTITY1 = Entity.builder(KEY1)
-      .setProperty("str", STR_VALUE)
-      .setProperty("bool", BOOL_VALUE)
-      .setProperty("partial1", new EntityValue(PARTIAL_ENTITY1))
-      .setProperty("list", LIST_VALUE2)
-      .build();
-  private static final Entity ENTITY2 = Entity.builder(ENTITY1)
-      .key(KEY2)
-      .removeProperty("str")
-      .setNullProperty("null")
-      .build();
-  private static final Entity ENTITY3 = Entity.builder(ENTITY1)
-      .key(KEY3)
-      .removeProperty("str")
-      .setProperty("null", NULL_VALUE)
-      .setEntityProperty("partial1", PARTIAL_ENTITY2)
-      .setEntityProperty("partial2", ENTITY2)
-      .build();
+      .remove("str").set("bool", true).set("list", LIST_VALUE1.get()).build();
+  private static final Entity ENTITY1 = Entity.builder(KEY1).set("str", STR_VALUE)
+      .set("bool", BOOL_VALUE).set("partial1", new EntityValue(PARTIAL_ENTITY1))
+      .set("list", LIST_VALUE2).build();
+  private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
+      .setNull("null").build();
+  private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
+      .set("null", NULL_VALUE).set("partial1", PARTIAL_ENTITY2).set("partial2", ENTITY2).build();
 
   private DatastoreServiceOptions options;
   private DatastoreService datastore;
@@ -72,10 +55,12 @@ public class DatastoreServiceTest {
   @Before
   public void setUp() {
     // TODO(ozarov): document that this test depends on a local gcd running.
-    // gcd.sh start dataset1
-    // reference: https://cloud.google.com/datastore/docs/tools/devserver
-    // Or even better, using a "GCE_HOME" param/env initiate and destroy the server
-    // before and after tests via ant or maven
+    // Unfortunately, the gcd tool is not bundled with the cloud SDK and need
+    // to be downloaded independently from
+    // https://cloud.google.com/datastore/docs/tools/devserver (b/16372095).
+    // To start the gcd run:
+    // gcd.sh create dataset1; gcd.sh start dataset1
+    // We should have an option to start the gcd from maven/ant.
     options = DatastoreServiceOptions.builder()
         .dataset(DATASET)
         .host("http://localhost:8080")
@@ -96,8 +81,8 @@ public void testNewTransactionCommit() {
     Transaction transaction = datastore.newTransaction();
     transaction.add(ENTITY3);
     Entity entity2 = Entity.builder(ENTITY2)
-        .clearProperties()
-        .setNullProperty("bla")
+        .clear()
+        .setNull("bla")
         .build();
     transaction.update(entity2);
     transaction.delete(KEY1);
@@ -137,7 +122,7 @@ public void testTransactionWithRead() {
     transaction = datastore.newTransaction();
     assertEquals(ENTITY3, transaction.get(KEY3));
     // update entity3 during the transaction
-    datastore.put(Entity.builder(ENTITY3).clearProperties().build());
+    datastore.put(Entity.builder(ENTITY3).clear().build());
     transaction.update(ENTITY2);
     try {
       transaction.commit();
@@ -148,15 +133,17 @@ public void testTransactionWithRead() {
     }
   }
 
+  @Test
+  public void testTransactionWithQuery() {
+    fail("not implemented");
+  }
+
   @Test
   public void testNewTransactionRollback() {
     Transaction transaction = datastore.newTransaction();
     transaction.add(ENTITY3);
-    Entity entity2 = Entity.builder(ENTITY2)
-        .clearProperties()
-        .setNullProperty("bla")
-        .setListProperty("list3", new StringValue("bla"), StringValue.builder("bla").build())
-        .build();
+    Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla")
+        .set("list3", new StringValue("bla"), StringValue.builder("bla").build()).build();
     transaction.update(entity2);
     transaction.delete(KEY1);
     transaction.rollback();
@@ -211,17 +198,10 @@ private void verifyNotUsable(DatastoreWriter writer) {
   @Test
   public void testNewBatchWriter() {
     BatchWriter batchWriter = datastore.newBatchWriter();
-    Entity entity1 = Entity.builder(ENTITY1).clearProperties().build();
-    Entity entity2 = Entity.builder(ENTITY2)
-        .clearProperties()
-        .setNullProperty("bla")
-        .build();
-    Entity entity4 = Entity.builder(KEY4)
-        .setProperty("value", new StringValue("value"))
-        .build();
-    Entity entity5 = Entity.builder(KEY5)
-        .setStringProperty("value", "value")
-        .build();
+    Entity entity1 = Entity.builder(ENTITY1).clear().build();
+    Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
+    Entity entity4 = Entity.builder(KEY4).set("value", new StringValue("value")).build();
+    Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
     batchWriter.add(entity4, entity5);
     batchWriter.put(ENTITY3, entity1, entity2);
     batchWriter.submit();
@@ -261,7 +241,12 @@ public void testNewBatchWriter() {
   }
 
   @Test
-  public void testRunQuery() {
+  public void testRunGqlQuery() {
+    fail("Not yet implemented");
+  }
+
+  @Test
+  public void testRunStructuredQuery() {
     fail("Not yet implemented");
   }
 
@@ -317,15 +302,15 @@ public void testGet() {
 
     entity = datastore.get(KEY1);
     assertEquals(ENTITY1, entity);
-    StringValue value1 = entity.property("str");
-    BooleanValue value2 = entity.property("bool");
-    ListValue value3 = entity.property("list");
+    StringValue value1 = entity.getValue("str");
+    BooleanValue value2 = entity.getValue("bool");
+    ListValue value3 = entity.getValue("list");
     assertEquals(value1, STR_VALUE);
     assertEquals(value2, BOOL_VALUE);
     assertEquals(value3, LIST_VALUE2);
-    assertEquals(PARTIAL_ENTITY1, entity.entityProperty("partial1"));
-    assertEquals(4, entity.propertyNames().size());
-    assertFalse(entity.hasProperty("bla"));
+    assertEquals(PARTIAL_ENTITY1, entity.getEntity("partial1"));
+    assertEquals(4, entity.names().size());
+    assertFalse(entity.contains("bla"));
   }
 
   @Test
@@ -338,18 +323,18 @@ public void testGetArray() {
     assertEquals(ENTITY2, result.next());
     Entity entity3 = result.next();
     assertEquals(ENTITY3, entity3);
-    assertTrue(entity3.isNullProperty("null"));
-    assertEquals(false, entity3.booleanProperty("bool"));
-    assertEquals(LIST_VALUE2.get(), entity3.listProperty("list"));
-    PartialEntity partial1 = entity3.entityProperty("partial1");
-    Entity partial2 = (Entity) entity3.entityProperty("partial2");
+    assertTrue(entity3.isNull("null"));
+    assertEquals(false, entity3.getBoolean("bool"));
+    assertEquals(LIST_VALUE2.get(), entity3.getList("list"));
+    PartialEntity partial1 = entity3.getEntity("partial1");
+    Entity partial2 = (Entity) entity3.getEntity("partial2");
     assertEquals(partial1, PARTIAL_ENTITY2);
     assertEquals(partial2, ENTITY2);
-    assertEquals(Value.Type.BOOLEAN, entity3.propertyType("bool"));
-    assertEquals(5, entity3.propertyNames().size());
-    assertFalse(entity3.hasProperty("bla"));
+    assertEquals(Value.Type.BOOLEAN, entity3.type("bool"));
+    assertEquals(5, entity3.names().size());
+    assertFalse(entity3.contains("bla"));
     try {
-      entity3.stringProperty("str");
+      entity3.getString("str");
       fail("Expecting a failure");
     } catch (DatastoreServiceException expected) {
       // expected - no such property
@@ -389,10 +374,7 @@ public void testUpdate() {
     }
     datastore.add(ENTITY3);
     assertEquals(ENTITY3, datastore.get(ENTITY3.key()));
-    Entity entity3 = Entity.builder(ENTITY3)
-        .clearProperties()
-        .setProperty("bla", new NullValue())
-        .build();
+    Entity entity3 = Entity.builder(ENTITY3).clear().set("bla", new NullValue()).build();
     assertNotEquals(ENTITY3, entity3);
     datastore.update(entity3);
     assertEquals(entity3, datastore.get(ENTITY3.key()));
@@ -406,10 +388,7 @@ public void testPut() {
     assertNull(keys.next());
     assertFalse(keys.hasNext());
 
-    Entity entity2 = Entity.builder(ENTITY2)
-        .clearProperties()
-        .setProperty("bla", new NullValue())
-        .build();
+    Entity entity2 = Entity.builder(ENTITY2).clear().set("bla", new NullValue()).build();
     assertNotEquals(ENTITY2, entity2);
     datastore.put(ENTITY3, ENTITY1, entity2);
     keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 1bb592b4eb93..c95356287ee1 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -7,6 +7,7 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Multimap;
+import com.google.gcloud.datastore.Query.ResultType;
 import com.google.gcloud.datastore.Value.Type;
 
 import org.junit.Test;
@@ -27,8 +28,14 @@ public class SerializationTest {
   private static final DateTime DATE_TIME1 = DateTime.now();
   private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world"));
   private static final Cursor CURSOR1 = Cursor.copyFrom(new byte[] {1,2});
-  private static final GqlQuery GQL1 =
-      GqlQuery.builder("select * from kind1 where name = @name and age > @1")
+  private static final Query GQL1 =
+      Query.builder("select * from kind1 where name = @name and age > @1")
+      .setArgument("name", "name1")
+      .addArgument(20)
+      .namespace("ns1")
+      .build();
+  private static final Query GQL2 =
+      Query.builder(ResultType.full(), "select * from kind1 where name = @name and age > @1")
       .setArgument("name", "name1")
       .addArgument(20)
       .namespace("ns1")
@@ -46,18 +53,18 @@ public class SerializationTest {
       DatastoreV1.Value.newBuilder().setBlobKeyValue("blob-key").setMeaning(18).build());
   private static final Entity ENTITY1 = Entity.builder(KEY1).build();
   private static final Entity ENTITY2 =
-      Entity.builder(KEY2).setProperty("null", new NullValue()).build();
+      Entity.builder(KEY2).set("null", new NullValue()).build();
   private static final Entity ENTITY3 = Entity.builder(KEY2)
-      .setProperty("p1", StringValue.builder("hi1").meaning(10).build())
-      .setProperty("p2", StringValue.builder("hi2").meaning(11).indexed(false).build())
-      .setProperty("p3", LongValue.builder(100).indexed(false).meaning(100).build())
-      .setBlobProperty("blob", BLOB1)
+      .set("p1", StringValue.builder("hi1").meaning(10).build())
+      .set("p2", StringValue.builder("hi2").meaning(11).indexed(false).build())
+      .set("p3", LongValue.builder(100).indexed(false).meaning(100).build())
+      .set("blob", BLOB1)
       .build();
   private static final PartialEntity EMBEDDED_ENTITY1 = ENTITY1;
   private static final PartialEntity EMBEDDED_ENTITY2 = ENTITY2;
   private static final PartialEntity EMBEDDED_ENTITY3 = PartialEntity.builder(INCOMPLETE_KEY1)
-      .setProperty("p1", STRING_VALUE)
-      .setProperty("p2", LongValue.builder(100).indexed(false).meaning(100).build())
+      .set("p1", STRING_VALUE)
+      .set("p2", LongValue.builder(100).indexed(false).meaning(100).build())
       .build();
   private static final EntityValue EMBEDDED_ENTITY_VALUE1 =
       new EntityValue(EMBEDDED_ENTITY1);
@@ -105,9 +112,8 @@ public void testValues() throws Exception {
   public void testTypes() throws Exception {
     Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
         ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3, DATE_TIME1,
-        BLOB1, CURSOR1, GQL1};
+        BLOB1, CURSOR1, GQL1, GQL2};
     for (Object obj : types) {
-      System.out.println("KOKO: " + obj);
       Object copy = serialiazeAndDeserialize(obj);
       assertEquals(obj, obj);
       assertEquals(obj, copy);

From ea5765bb5df9d86277c2e452746ead11531329c5 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 11 Dec 2014 17:16:46 -0800
Subject: [PATCH 048/732] structured query - work in progress

---
 .../com/google/gcloud/datastore/GqlQuery.java |  45 ++++++-
 .../com/google/gcloud/datastore/Query.java    |   4 +-
 .../gcloud/datastore/StructuredQuery.java     | 125 ++++++++++++------
 .../datastore/DatastoreServiceTest.java       |  19 ++-
 4 files changed, 148 insertions(+), 45 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 5e0a5c89323f..f8d098d187c5 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -3,7 +3,10 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.primitives.Booleans;
 import com.google.common.primitives.Doubles;
 import com.google.common.primitives.Longs;
@@ -23,7 +26,7 @@
  *
  * @see GQL Reference
  */
-final class GqlQuery extends Query {
+public final class GqlQuery extends Query {
 
   private static final long serialVersionUID = 5988280590929540569L;
 
@@ -32,7 +35,7 @@ final class GqlQuery extends Query {
   private final transient ImmutableList nameArgs;
   private final transient ImmutableList numberArgs;
 
-  private static final class Argument extends Serializable {
+  static final class Argument extends Serializable {
 
     private static final long serialVersionUID = 1976895435257636275L;
 
@@ -42,16 +45,20 @@ private static final class Argument extends Serializable value) {
       this.name = name;
-      this.value = value;
+      this.value = checkNotNull(value);
       cursor = null;
     }
 
+    Object cursorOrValue() {
+      return MoreObjects.firstNonNull(cursor, value);
+    }
+
     String name() {
       return name;
     }
@@ -268,6 +275,36 @@ private GqlQuery(Builder builder) {
     numberArgs = ImmutableList.copyOf(builder.numberArgs);
   }
 
+  public String queryString() {
+    return queryString;
+  }
+
+  public boolean allowLiteral() {
+    return allowLiteral;
+  }
+
+  /**
+   * Returns an immutable map of named arguments.
+   */
+  public Map nameArgs() {
+    ImmutableMap.Builder builder = ImmutableSortedMap.naturalOrder();
+    for (Argument argument : nameArgs) {
+      builder.put(argument.name(), argument.cursorOrValue());
+    }
+    return builder.build();
+  }
+
+  /**
+   * Returns an immutable list of numbered arguments (using original order).
+   */
+  public List numberArgs() {
+    ImmutableList.Builder builder = ImmutableList.builder();
+    for (Argument argument : numberArgs) {
+      builder.add(argument.cursorOrValue());
+    }
+    return builder.build();
+  }
+
   @Override
   public int hashCode() {
     return Objects.hash(namespace(), queryString, allowLiteral, nameArgs, numberArgs);
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index d0bcaf70de79..9ef104ee8d25 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -119,7 +119,7 @@ public static  GqlQuery.Builder builder(ResultType resultType, String g
   /**
    * Returns a new structured query builder.
    */
-  public static StructuredQuery.FullBuilder builder() {
-    return new StructuredQuery.FullBuilder();
+  public static StructuredQuery.Builder builder() {
+    return new StructuredQuery.Builder();
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 890dca19de4b..ce5c2385899b 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -1,80 +1,108 @@
 package com.google.gcloud.datastore;
 
+import com.google.api.client.util.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.collect.ImmutableSet;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 
-final class StructuredQuery extends Query {
+import java.util.Objects;
+
+public final class StructuredQuery extends Query {
 
   private static final long serialVersionUID = 546838955624019594L;
 
+  private final transient boolean keysOnly;
+  private final transient String kind;
+  private final transient ImmutableSet projections;
+  private final transient Cursor startCursor;
+  private final transient Cursor endCursor;
+  private final transient Integer offset;
+  private final transient Integer limit;
 
-  static class BaseBuilder> {
 
-    private String kind;
+  static class Builder {
+
     private String namespace;
+    private String kind;
     private Cursor startCursor;
     private Cursor endCursor;
     private Integer offset;
     private Integer limit;
 
-
-    protected B self() {
-      return (B) this;
+    public Builder namespace(String namespace) {
+      this.namespace = namespace;
+      return this;
     }
 
-    public B kind(String kind) {
+    public Builder kind(String kind) {
       this.kind = kind;
-      return self();
+      return this;
     }
 
-    public B namespace(String namespace) {
-      this.namespace = namespace;
-      return self();
-    }
-
-    public B startCursor(Cursor startCursor) {
+    public Builder startCursor(Cursor startCursor) {
       this.startCursor = startCursor;
-      return self();
+      return this;
     }
 
-    public B encCursor(Cursor endCursor) {
+    public Builder encCursor(Cursor endCursor) {
       this.endCursor = endCursor;
-      return self();
+      return this;
     }
 
-    public B offset(int offset) {
+    public Builder offset(Integer offset) {
+      Preconditions.checkArgument(offset == null || offset >= 0, "offset must not be negative");
       this.offset = offset;
-      return self();
+      return this;
     }
 
-    public B limit(int limit) {
+    public Builder limit(Integer limit) {
+      Preconditions.checkArgument(limit == null || offset > 0, "limit must be positive");
       this.limit = limit;
-      return self();
+      return this;
     }
-  }
-
-  public static final class FullBuilder extends BaseBuilder {
 
-  }
+    public StructuredQuery full() {
+      return new StructuredQuery<>(ResultType.full(), this, false);
+    }
 
-  public static final class KeyOnlyBuilder extends BaseBuilder {
+    public StructuredQuery projection(String projection, String... other) {
+      ImmutableSet projections =
+          ImmutableSet.builder().add(projection).add(other).build();
+      return new StructuredQuery<>(ResultType.projection(), this, projections);
+    }
 
+    public StructuredQuery keyOnly() {
+      return new StructuredQuery<>(ResultType.keyOnly(), this, true);
+    }
   }
 
-  public static final class ProjectionBuilder
-      extends BaseBuilder {
-
+  private StructuredQuery(ResultType resultType, Builder builder,
+      ImmutableSet projections) {
+    super(resultType, builder.namespace);
+    kind = builder.kind;
+    startCursor = builder.startCursor;
+    endCursor = builder.endCursor;
+    offset = builder.offset;
+    limit = builder.limit;
+    this.projections = projections;
+    keysOnly = false;
   }
 
-  private StructuredQuery(ResultType resultType, String namespace) {
-    super(resultType, namespace);
+  private StructuredQuery(ResultType resultType, Builder builder, boolean keysOnly) {
+    super(resultType, builder.namespace);
+    kind = builder.kind;
+    startCursor = builder.startCursor;
+    endCursor = builder.endCursor;
+    offset = builder.offset;
+    limit = builder.limit;
+    this.keysOnly = keysOnly;
+    projections = ImmutableSet.of();
   }
 
   @Override
   public int hashCode() {
-    // implement
-    return 0;
+    return Objects.hash(namespace(), kind, startCursor, endCursor, offset, limit);
   }
 
   @Override
@@ -82,8 +110,16 @@ public boolean equals(Object obj) {
     if (obj == this) {
       return true;
     }
-    // implement
-    return false;
+    if (!(obj instanceof StructuredQuery)) {
+      return false;
+    }
+    StructuredQuery other = (StructuredQuery) obj;
+    return Objects.equals(namespace(), other.namespace())
+        && Objects.equals(kind, other.kind)
+        && Objects.equals(startCursor, other.startCursor)
+        && Objects.equals(endCursor, other.endCursor)
+        && Objects.equals(offset, other.offset)
+        && Objects.equals(limit, other.limit);
   }
 
   @Override
@@ -104,8 +140,23 @@ protected Object fromPb(ResultType resultType, String namespace, byte[] bytes
 
   @Override
   protected DatastoreV1.Query toPb() {
-    // TODO Auto-generated method stub
-    return null;
+    DatastoreV1.Query.Builder queryPb = DatastoreV1.Query.newBuilder();
+    if (kind != null) {
+      queryPb.addKindBuilder().setName(kind);
+    }
+    if (startCursor != null) {
+      queryPb.setStartCursor(startCursor.byteString());
+    }
+    if (endCursor != null) {
+      queryPb.setEndCursor(endCursor.byteString());
+    }
+    if (offset != null) {
+      queryPb.setOffset(offset);
+    }
+    if (limit != null) {
+      queryPb.setLimit(limit);
+    }
+    return queryPb.build();
   }
 
   static  StructuredQuery fromPb(ResultType resultType, String namespace,
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 0e22bd8ce195..a9fe6e53c005 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -241,12 +241,27 @@ public void testNewBatchWriter() {
   }
 
   @Test
-  public void testRunGqlQuery() {
+  public void testRunGqlQueryNoCasting() {
     fail("Not yet implemented");
   }
 
   @Test
-  public void testRunStructuredQuery() {
+  public void testRunGqlQueryWithCasting() {
+    fail("Not yet implemented");
+  }
+
+  @Test
+  public void testRunStructuredQueryFull() {
+    fail("Not yet implemented");
+  }
+
+  @Test
+  public void testRunStructuredQueryProjection() {
+    fail("Not yet implemented");
+  }
+
+  @Test
+  public void testRunStructuredQueryKeysOnly() {
     fail("Not yet implemented");
   }
 

From 393682e16733b65bbc7809199711dfb255ae4c93 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 11 Dec 2014 21:10:00 -0800
Subject: [PATCH 049/732] Fix compilation error visible by maven

---
 .classpath                                    | 10 ----
 .settings/org.eclipse.jdt.core.prefs          |  3 --
 pom.xml                                       |  2 +-
 .../com/google/gcloud/datastore/Value.java    | 47 ++++++++-----------
 4 files changed, 20 insertions(+), 42 deletions(-)

diff --git a/.classpath b/.classpath
index d6cf6121af66..9ed6ee4d0713 100644
--- a/.classpath
+++ b/.classpath
@@ -23,15 +23,5 @@
 		
 	
 	
-	
-		
-			
-		
-	
-	
-		
-			
-		
-	
 	
 
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
index 62f72418e536..0192f1f052aa 100644
--- a/.settings/org.eclipse.jdt.core.prefs
+++ b/.settings/org.eclipse.jdt.core.prefs
@@ -8,8 +8,6 @@ org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonN
 org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
 org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
 org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
-org.eclipse.jdt.core.compiler.compliance=1.7
 org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
 org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
 org.eclipse.jdt.core.compiler.problem.comparingIdentical=error
@@ -92,4 +90,3 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
 org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=warning
 org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
 org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
-org.eclipse.jdt.core.compiler.source=1.7
diff --git a/pom.xml b/pom.xml
index d3431c1584fd..3a573d91459f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -53,7 +53,7 @@
       
         org.apache.maven.plugins
         maven-compiler-plugin
-        2.5.1
+        3.1
         
                 1.7
                 1.7
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index 9c569ae24de4..4686624adf54 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -40,85 +40,72 @@ public enum Type {
     /**
      * Represents a {@code null} value.
      */
-    NULL(NullValue.MARSHALLER, NullValue.MARSHALLER),
+    NULL(NullValue.MARSHALLER),
 
     /**
      * Represents a {@code string} value.
      */
-    STRING(StringValue.MARSHALLER, StringValue.MARSHALLER),
+    STRING(StringValue.MARSHALLER),
 
     /**
      * Represents an entity ({@link PartialEntity} or {@link Entity}) value.
      */
-    ENTITY(EntityValue.MARSHALLER, EntityValue.MARSHALLER),
+    ENTITY(EntityValue.MARSHALLER),
 
     /**
      * Represents a {@code list} of {@link Value}s.
      */
-    LIST(ListValue.MARSHALLER, ListValue.MARSHALLER),
+    LIST(ListValue.MARSHALLER),
 
     /**
      * Represents a {@code key} as a value.
      */
-    KEY(KeyValue.MARSHALLER, KeyValue.MARSHALLER),
+    KEY(KeyValue.MARSHALLER),
 
     /**
      * Represents a {@code long} value.
      */
-    LONG(LongValue.MARSHALLER, LongValue.MARSHALLER),
+    LONG(LongValue.MARSHALLER),
 
     /**
      * Represents a {@code double} value.
      */
-    DOUBLE(DoubleValue.MARSHALLER, DoubleValue.MARSHALLER),
+    DOUBLE(DoubleValue.MARSHALLER),
 
     /**
      * Represents a {@code boolean} value.
      */
-    BOOLEAN(BooleanValue.MARSHALLER, BooleanValue.MARSHALLER),
+    BOOLEAN(BooleanValue.MARSHALLER),
 
     /**
      * Represents a {@link DateTime} value.
      */
-    DATE_TIME(DateTimeValue.MARSHALLER, DateTimeValue.MARSHALLER),
+    DATE_TIME(DateTimeValue.MARSHALLER),
 
     /**
      * Represents a {@link Blob} value.
      */
-    BLOB(BlobValue.MARSHALLER, BlobValue.MARSHALLER),
+    BLOB(BlobValue.MARSHALLER),
 
     /**
      * Represents a raw/unparsed value.
      */
-    RAW_VALUE(RawValue.MARSHALLER, RawValue.MARSHALLER);
+    RAW_VALUE(RawValue.MARSHALLER);
 
 
-    @SuppressWarnings("rawtypes") private final BuilderFactory builderFactory;
     @SuppressWarnings("rawtypes") private final Marshaller marshaller;
 
-    , B extends Builder> Type(Marshaller marshaller,
-        BuilderFactory builderFactory) {
+    , B extends Builder> Type(Marshaller marshaller) {
       this.marshaller = marshaller;
-      this.builderFactory = builderFactory;
       int fieldId = marshaller.getProtoFieldId();
       if (fieldId > 0) {
         DESCRIPTOR_TO_TYPE_MAP.put(fieldId, this);
       }
     }
 
-    , B extends Builder> Marshaller getMarshaller() {
+    Marshaller getMarshaller() {
       return marshaller;
     }
-
-    , B extends Builder> BuilderFactory
-        getBuilderFactory() {
-      return builderFactory;
-    }
-  }
-
-  interface BuilderFactory, B extends Builder> {
-
-    B newBuilder(V value);
   }
 
   interface Builder, B extends Builder> {
@@ -142,6 +129,10 @@ interface Builder, B extends Builder> {
     P build();
   }
 
+  interface BuilderFactory, B extends Builder> {
+    B newBuilder(V value);
+  }
+
   interface Marshaller, B extends Builder> {
 
     B fromProto(DatastoreV1.Value proto);
@@ -314,9 +305,9 @@ public boolean equals(Object obj) {
   }
 
   @Override
-  @SuppressWarnings({"unchecked", "rawtypes"})
+  @SuppressWarnings("unchecked")
   protected DatastoreV1.Value toPb() {
-    return type().getMarshaller().toProto((Value) this);
+    return type().getMarshaller().toProto(this);
   }
 
   static Value fromPb(DatastoreV1.Value proto) {

From 81d22393f43ece2d3cf14f83ad14403ae6a6983a Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 11 Dec 2014 23:21:27 -0800
Subject: [PATCH 050/732] structured query - work in progress

---
 .../com/google/gcloud/datastore/BaseKey.java  |  26 ++-
 .../com/google/gcloud/datastore/GqlQuery.java |  18 ++
 .../java/com/google/gcloud/datastore/Key.java |  26 +--
 .../google/gcloud/datastore/PartialKey.java   |  26 +--
 .../com/google/gcloud/datastore/Query.java    |  18 --
 .../gcloud/datastore/StructuredQuery.java     | 208 ++++++++++++++++--
 .../gcloud/datastore/SerializationTest.java   |   4 +-
 7 files changed, 234 insertions(+), 92 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index 09edf60afeac..4b4a33aa404e 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -11,6 +11,7 @@
 
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Base class for keys.
@@ -147,6 +148,26 @@ public String kind() {
     return kind;
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(dataset(), namespace(), ancestors(), leaf());
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+    if (!(obj instanceof BaseKey)) {
+      return false;
+    }
+    PartialKey other = (PartialKey) obj;
+    return Objects.equals(dataset(), other.dataset())
+        && Objects.equals(namespace(), other.namespace())
+        && Objects.equals(ancestors(), other.ancestors())
+        && Objects.equals(leaf(), other.leaf());
+  }
+
   @Override
   protected DatastoreV1.Key toPb() {
     DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder();
@@ -163,9 +184,10 @@ protected DatastoreV1.Key toPb() {
     for (PathElement pathEntry : ancestors) {
       keyPb.addPathElement(pathEntry.toPb());
     }
-    addLeaf(keyPb);
+    PathElement leaf = leaf();
+    keyPb.addPathElement(leaf.toPb());
     return keyPb.build();
   }
 
-  protected abstract void addLeaf(DatastoreV1.Key.Builder keyPb);
+  protected abstract PathElement leaf();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index f8d098d187c5..d7c481d613ee 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -374,4 +374,22 @@ static  GqlQuery fromPb(ResultType resultType, String namespace,
     }
     return builder.build();
   }
+
+  /**
+   * Returns a new GQL query builder.
+   *
+   * @see GQL Reference
+   */
+  public static GqlQuery.Builder builder(String gql) {
+    return builder(ResultType.unknown(), gql);
+  }
+
+  /**
+   * Returns a new GQL query builder.
+   *
+   * @see GQL Reference
+   */
+  public static  GqlQuery.Builder builder(ResultType resultType, String gql) {
+    return new GqlQuery.Builder<>(resultType, gql);
+  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index 9687ea9e830e..272fc6e4d7f4 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -11,7 +11,6 @@
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
-import java.util.Objects;
 
 /**
  * A key that is guaranteed to be complete and could be used to reference a
@@ -142,29 +141,8 @@ public static Key fromUrlSafe(String urlSafe) {
   }
 
   @Override
-  public int hashCode() {
-    return Objects.hash(dataset(), namespace(), ancestors(), kind(), leaf);
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-    if (!(obj instanceof Key)) {
-      return false;
-    }
-    Key other = (Key) obj;
-    return Objects.equals(dataset(), other.dataset())
-        && Objects.equals(namespace(), other.namespace())
-        && Objects.equals(ancestors(), other.ancestors())
-        && Objects.equals(kind(), other.kind())
-        && Objects.equals(leaf, other.leaf);
-  }
-
-  @Override
-  protected void addLeaf(DatastoreV1.Key.Builder keyPb) {
-    keyPb.addPathElement(leaf.toPb());
+  protected PathElement leaf() {
+    return leaf;
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java
index 74f5d5157bf0..f7e6c00374c8 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java
@@ -5,7 +5,6 @@
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.util.List;
-import java.util.Objects;
 
 /**
  * A partial key (without a name or id).
@@ -46,29 +45,8 @@ public Key newKey(long id) {
   }
 
   @Override
-  public int hashCode() {
-    return Objects.hash(dataset(), namespace(), ancestors(), kind());
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-    if (!(obj instanceof PartialKey) || !PartialKey.class.equals(obj.getClass())) {
-      return false;
-    }
-
-    PartialKey other = (PartialKey) obj;
-    return Objects.equals(dataset(), other.dataset())
-        && Objects.equals(namespace(), other.namespace())
-        && Objects.equals(ancestors(), other.ancestors())
-        && Objects.equals(kind(), other.kind());
-  }
-
-  @Override
-  protected void addLeaf(DatastoreV1.Key.Builder keyPb) {
-    keyPb.addPathElement(new PathElement(kind()).toPb());
+  protected PathElement leaf() {
+    return new PathElement(kind());
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 9ef104ee8d25..d1c29d189599 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -98,24 +98,6 @@ protected abstract Object fromPb(ResultType resultType, String namespace, byt
   protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead,
       ByteString batchCursor);
 
-  /**
-   * Returns a new GQL query builder.
-   *
-   * @see GQL Reference
-   */
-  public static GqlQuery.Builder builder(String gql) {
-    return builder(ResultType.unknown(), gql);
-  }
-
-  /**
-   * Returns a new GQL query builder.
-   *
-   * @see GQL Reference
-   */
-  public static  GqlQuery.Builder builder(ResultType resultType, String gql) {
-    return new GqlQuery.Builder<>(resultType, gql);
-  }
-
   /**
    * Returns a new structured query builder.
    */
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index ce5c2385899b..b5e6a98fe1dc 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -1,12 +1,23 @@
 package com.google.gcloud.datastore;
 
+import static com.google.api.client.util.Preconditions.checkNotNull;
+
 import com.google.api.client.util.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 
+import java.io.Serializable;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.Objects;
+import java.util.Set;
 
 public final class StructuredQuery extends Query {
 
@@ -14,13 +25,160 @@ public final class StructuredQuery extends Query {
 
   private final transient boolean keysOnly;
   private final transient String kind;
-  private final transient ImmutableSet projections;
+  private final transient ImmutableList projection;
+  private final transient Filter filter;
+  private final transient ImmutableList groupBy;
+  private final transient ImmutableList orderBy;
   private final transient Cursor startCursor;
   private final transient Cursor endCursor;
   private final transient Integer offset;
   private final transient Integer limit;
 
 
+  public static final class Filter implements Serializable {
+
+    private static final long serialVersionUID = -4514695915258598597L;
+
+    private final String property;
+    private final Operator operator;
+    private final Value value;
+    private final Filter next;
+
+    public enum Operator {
+      LESS_THAN,
+      LESS_THAN_OR_EQUAL,
+      GREATER_THAN,
+      GREATER_THAN_OR_EQUAL,
+      EQUAL,
+      HAS_ANCESTOR,
+      AND
+    }
+
+    private Filter(String property, Operator operator, Value value, Filter next) {
+      this.property = checkNotNull(property);
+      this.operator = checkNotNull(operator);
+      this.value = value;
+      this.next = next;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(property, operator, value, next);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof Filter)) {
+        return false;
+      }
+      Filter other = (Filter) obj;
+      return property.equals(other.property)
+          && operator.equals(other.operator)
+          && Objects.equals(value, other.value)
+          && Objects.equals(next, other.next);
+    }
+
+    public static Filter le(String property, Value value) {
+      return new Filter(property, Operator.LESS_THAN, value, null);
+    }
+
+    public static Filter lte(String property, Value value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, value, null);
+    }
+
+    public static Filter gt(String property, Value value) {
+      return new Filter(property, Operator.GREATER_THAN, value, null);
+    }
+
+    public static Filter gte(String property, Value value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, value, null);
+    }
+
+    public static Filter eq(String property, Value value) {
+      return new Filter(property, Operator.EQUAL, value, null);
+    }
+
+    public static Filter hasAncestor(String property) {
+      return new Filter(property, Operator.HAS_ANCESTOR, null, null);
+    }
+
+    public static Filter and(Filter first, Filter... other) {
+      Set combined = new LinkedHashSet<>();
+      for (Filter f : Iterables.concat(Collections.singleton(first), Arrays.asList(other))) {
+        while (f != null) {
+          combined.add(new Filter(f.property, f.operator, f.value, null));
+          f = f.next;
+        }
+      }
+      ArrayDeque stack = new ArrayDeque<>(combined.size());
+      for (Filter f : combined) {
+        stack.push(f);
+      }
+      while (true) {
+        Filter f1 = stack.pop();
+        if (stack.isEmpty()) {
+          return f1;
+        }
+        Filter f2 = stack.pop();
+        stack.push(new Filter(f2.property, f2.operator, f2.value, f1));
+      }
+    }
+  }
+
+  public static final class OrderBy implements Serializable {
+
+    private static final long serialVersionUID = 4091186784814400031L;
+
+    private final String property;
+    private final Direction direction;
+
+    public enum Direction {
+      ASC, DESC
+    }
+
+    public OrderBy(String property, Direction direction) {
+      this.property = checkNotNull(property);
+      this.direction = checkNotNull(direction);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(property, direction);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof OrderBy)) {
+        return false;
+      }
+      OrderBy other = (OrderBy) obj;
+      return property.equals(other.property)
+          && direction.equals(other.direction);
+    }
+
+    public String property() {
+      return property;
+    }
+
+    public Direction direction() {
+      return direction;
+    }
+
+    public static OrderBy asc(String property) {
+      return new OrderBy(property, OrderBy.Direction.ASC);
+    }
+
+    public static OrderBy desc(String property) {
+      return new OrderBy(property, OrderBy.Direction.DESC);
+    }
+  }
+
   static class Builder {
 
     private String namespace;
@@ -62,6 +220,7 @@ public Builder limit(Integer limit) {
       return this;
     }
 
+    /*
     public StructuredQuery full() {
       return new StructuredQuery<>(ResultType.full(), this, false);
     }
@@ -75,34 +234,33 @@ public StructuredQuery projection(String projection, String... ot
     public StructuredQuery keyOnly() {
       return new StructuredQuery<>(ResultType.keyOnly(), this, true);
     }
+    */
   }
 
-  private StructuredQuery(ResultType resultType, Builder builder,
-      ImmutableSet projections) {
-    super(resultType, builder.namespace);
-    kind = builder.kind;
-    startCursor = builder.startCursor;
-    endCursor = builder.endCursor;
-    offset = builder.offset;
-    limit = builder.limit;
-    this.projections = projections;
-    keysOnly = false;
-  }
 
-  private StructuredQuery(ResultType resultType, Builder builder, boolean keysOnly) {
-    super(resultType, builder.namespace);
-    kind = builder.kind;
-    startCursor = builder.startCursor;
-    endCursor = builder.endCursor;
-    offset = builder.offset;
-    limit = builder.limit;
+
+  private StructuredQuery(ResultType resultType, boolean keysOnly, String namespace, String kind,
+      Cursor startCursor, Cursor endCursor, Integer offset, Integer limit,
+      ImmutableList projection, Filter filter, ImmutableList groupBy,
+      ImmutableList orderBy) {
+    super(resultType, namespace);
     this.keysOnly = keysOnly;
-    projections = ImmutableSet.of();
+    this.kind = kind;
+    this.startCursor = startCursor;
+    this.endCursor = endCursor;
+    this.offset = offset;
+    this.limit = limit;
+    this.projection = projection;
+    this.filter = filter;
+    this.groupBy = groupBy;
+    this.orderBy = orderBy;
+    Preconditions.checkState(keysOnly == false || projection.isEmpty() && groupBy.isEmpty(),
+        "projection or group by are not applicable for keys-only queries");
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(namespace(), kind, startCursor, endCursor, offset, limit);
+    return Objects.hash(namespace(), keysOnly, kind, startCursor, endCursor, offset, limit, projection, filter, groupBy, orderBy);
   }
 
   @Override
@@ -119,7 +277,11 @@ public boolean equals(Object obj) {
         && Objects.equals(startCursor, other.startCursor)
         && Objects.equals(endCursor, other.endCursor)
         && Objects.equals(offset, other.offset)
-        && Objects.equals(limit, other.limit);
+        && Objects.equals(limit, other.limit)
+        && Objects.equals(projection, other.projection)
+        && Objects.equals(filter, other.filter)
+        && Objects.equals(groupBy, other.groupBy)
+        && Objects.equals(orderBy, other.orderBy);
   }
 
   @Override
@@ -156,11 +318,13 @@ protected DatastoreV1.Query toPb() {
     if (limit != null) {
       queryPb.setLimit(limit);
     }
+    // TODO: projection, filter, groupBy, orderBy (or keys-only)
     return queryPb.build();
   }
 
   static  StructuredQuery fromPb(ResultType resultType, String namespace,
       DatastoreV1.Query queryPb) {
+    // TODO: implement
     return null;
   }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index c95356287ee1..eb63574ec021 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -29,13 +29,13 @@ public class SerializationTest {
   private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world"));
   private static final Cursor CURSOR1 = Cursor.copyFrom(new byte[] {1,2});
   private static final Query GQL1 =
-      Query.builder("select * from kind1 where name = @name and age > @1")
+      GqlQuery.builder("select * from kind1 where name = @name and age > @1")
       .setArgument("name", "name1")
       .addArgument(20)
       .namespace("ns1")
       .build();
   private static final Query GQL2 =
-      Query.builder(ResultType.full(), "select * from kind1 where name = @name and age > @1")
+      GqlQuery.builder(ResultType.full(), "select * from kind1 where name = @name and age > @1")
       .setArgument("name", "name1")
       .addArgument(20)
       .namespace("ns1")

From 41422a9d39c4e8c49e1cfa98d32ee2e36ba38c95 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Fri, 12 Dec 2014 09:21:51 -0800
Subject: [PATCH 051/732] Adding static of method for Values

---
 .../google/gcloud/datastore/BaseEntity.java   | 33 ++++++++++-----
 .../google/gcloud/datastore/BlobValue.java    |  4 ++
 .../google/gcloud/datastore/BooleanValue.java |  4 ++
 .../gcloud/datastore/DateTimeValue.java       |  4 ++
 .../google/gcloud/datastore/DoubleValue.java  |  4 ++
 .../google/gcloud/datastore/EntityValue.java  |  4 ++
 .../com/google/gcloud/datastore/KeyValue.java |  4 ++
 .../google/gcloud/datastore/ListValue.java    |  8 ++++
 .../google/gcloud/datastore/LongValue.java    |  4 ++
 .../google/gcloud/datastore/NullValue.java    |  4 ++
 .../com/google/gcloud/datastore/RawValue.java |  4 ++
 .../google/gcloud/datastore/StringValue.java  |  4 ++
 .../gcloud/datastore/StructuredQuery.java     | 40 +++++++++++++++++++
 .../datastore/DatastoreServiceTest.java       | 14 +++----
 .../gcloud/datastore/SerializationTest.java   | 28 ++++++-------
 15 files changed, 129 insertions(+), 34 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 010c97dc256e..606630a5b10c 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -1,5 +1,16 @@
 package com.google.gcloud.datastore;
 
+import static com.google.gcloud.datastore.BlobValue.of;
+import static com.google.gcloud.datastore.BooleanValue.of;
+import static com.google.gcloud.datastore.DateTimeValue.of;
+import static com.google.gcloud.datastore.DoubleValue.of;
+import static com.google.gcloud.datastore.EntityValue.of;
+import static com.google.gcloud.datastore.KeyValue.of;
+import static com.google.gcloud.datastore.ListValue.of;
+import static com.google.gcloud.datastore.LongValue.of;
+import static com.google.gcloud.datastore.NullValue.of;
+import static com.google.gcloud.datastore.StringValue.of;
+
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableSortedMap;
 import com.google.gcloud.datastore.Value.Type;
@@ -58,57 +69,57 @@ public B set(String name, Value value) {
     }
 
     public B setNull(String name) {
-      properties.put(name, new NullValue());
+      properties.put(name, of());
       return self();
     }
 
     public B set(String name, String value) {
-      properties.put(name, new StringValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
     public B set(String name, long value) {
-      properties.put(name, new LongValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
     public B set(String name, double value) {
-      properties.put(name, new DoubleValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
     public B set(String name, boolean value) {
-      properties.put(name, new BooleanValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
     public B set(String name, DateTime value) {
-      properties.put(name, new DateTimeValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
     public B set(String name, Key value) {
-      properties.put(name, new KeyValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
     public B set(String name, PartialEntity value) {
-      properties.put(name, new EntityValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
     public B set(String name, List> values) {
-      properties.put(name, new ListValue(values));
+      properties.put(name, of(values));
       return self();
     }
 
     public B set(String name, Value... value) {
-      properties.put(name, new ListValue(Arrays.asList(value)));
+      properties.put(name, of(Arrays.asList(value)));
       return self();
     }
 
     public B set(String name, Blob value) {
-      properties.put(name, new BlobValue(value));
+      properties.put(name, of(value));
       return self();
     }
 
diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java
index 8d369c9eb447..73671e167657 100644
--- a/src/main/java/com/google/gcloud/datastore/BlobValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java
@@ -57,6 +57,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static BlobValue of(Blob blob) {
+    return new BlobValue(blob);
+  }
+
   public static Builder builder(Blob blob) {
     return new Builder().set(blob);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
index f6323bb3f794..86e080b096fb 100644
--- a/src/main/java/com/google/gcloud/datastore/BooleanValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
@@ -57,6 +57,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static BooleanValue of(boolean value) {
+    return new BooleanValue(value);
+  }
+
   public static Builder builder(boolean value) {
     return new Builder().set(value);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
index a178c71a9665..fff1b35d7f29 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
@@ -58,6 +58,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static DateTimeValue of(DateTime dateTime) {
+    return new DateTimeValue(dateTime);
+  }
+
   public static Builder builder(DateTime dateTime) {
     return new Builder().set(dateTime);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
index 749a2a9b7bd5..5d5b510739f0 100644
--- a/src/main/java/com/google/gcloud/datastore/DoubleValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
@@ -57,6 +57,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static DoubleValue of(double value) {
+    return new DoubleValue(value);
+  }
+
   public static Builder builder(double value) {
     return new Builder().set(value);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java
index 243c1bac7be9..061cf6c57fa5 100644
--- a/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -66,6 +66,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static EntityValue of(PartialEntity entity) {
+    return new EntityValue(entity);
+  }
+
   public static Builder builder(PartialEntity entity) {
     return new Builder().set(entity).indexed(false);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java
index b9aab3134a86..a97399b5006c 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyValue.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java
@@ -57,6 +57,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static KeyValue of(Key key) {
+    return new KeyValue(key);
+  }
+
   public static Builder builder(Key key) {
     return new Builder().set(key);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java
index b9d7aa66c21c..5af230d768b1 100644
--- a/src/main/java/com/google/gcloud/datastore/ListValue.java
+++ b/src/main/java/com/google/gcloud/datastore/ListValue.java
@@ -116,6 +116,14 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static ListValue of(List> values) {
+    return new ListValue(values);
+  }
+
+  public static ListValue of(Value first, Value... other) {
+    return new ListValue(first, other);
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/src/main/java/com/google/gcloud/datastore/LongValue.java
index 9107223c7af4..4ab977ed96da 100644
--- a/src/main/java/com/google/gcloud/datastore/LongValue.java
+++ b/src/main/java/com/google/gcloud/datastore/LongValue.java
@@ -57,6 +57,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static LongValue of(long value) {
+    return new LongValue(value);
+  }
+
   public static Builder builder(long value) {
     return new Builder().set(value);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java
index cf4f463095d0..af81c017bc72 100644
--- a/src/main/java/com/google/gcloud/datastore/NullValue.java
+++ b/src/main/java/com/google/gcloud/datastore/NullValue.java
@@ -63,6 +63,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static NullValue of() {
+    return new NullValue();
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/src/main/java/com/google/gcloud/datastore/RawValue.java b/src/main/java/com/google/gcloud/datastore/RawValue.java
index cf5852d46c13..d5fc455df4a9 100644
--- a/src/main/java/com/google/gcloud/datastore/RawValue.java
+++ b/src/main/java/com/google/gcloud/datastore/RawValue.java
@@ -55,6 +55,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  static RawValue of(DatastoreV1.Value valuePb) {
+    return new RawValue(valuePb);
+  }
+
   static Builder builder(DatastoreV1.Value valuePb) {
     Builder builder = new Builder();
     if (valuePb.hasIndexed()) {
diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java
index 6434b6a86f40..dad18654c8bd 100644
--- a/src/main/java/com/google/gcloud/datastore/StringValue.java
+++ b/src/main/java/com/google/gcloud/datastore/StringValue.java
@@ -61,6 +61,10 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
+  public static StringValue of(String value) {
+    return new StringValue(value);
+  }
+
   public static Builder builder(String value) {
     return new Builder().set(value);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index b5e6a98fe1dc..77ee46e1438b 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -1,6 +1,13 @@
 package com.google.gcloud.datastore;
 
 import static com.google.api.client.util.Preconditions.checkNotNull;
+import static com.google.gcloud.datastore.BlobValue.of;
+import static com.google.gcloud.datastore.BooleanValue.of;
+import static com.google.gcloud.datastore.DateTimeValue.of;
+import static com.google.gcloud.datastore.DoubleValue.of;
+import static com.google.gcloud.datastore.KeyValue.of;
+import static com.google.gcloud.datastore.LongValue.of;
+import static com.google.gcloud.datastore.StringValue.of;
 
 import com.google.api.client.util.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
@@ -51,6 +58,7 @@ public enum Operator {
       GREATER_THAN_OR_EQUAL,
       EQUAL,
       HAS_ANCESTOR,
+      IS_NULL,
       AND
     }
 
@@ -85,6 +93,34 @@ public static Filter le(String property, Value value) {
       return new Filter(property, Operator.LESS_THAN, value, null);
     }
 
+    public static Filter le(String property, String value) {
+      return new Filter(property, Operator.LESS_THAN, of(value), null);
+    }
+
+    public static Filter le(String property, long value) {
+      return new Filter(property, Operator.LESS_THAN, of(value), null);
+    }
+
+    public static Filter le(String property, double value) {
+      return new Filter(property, Operator.LESS_THAN, of(value), null);
+    }
+
+    public static Filter le(String property, boolean value) {
+      return new Filter(property, Operator.LESS_THAN, of(value), null);
+    }
+
+    public static Filter le(String property, DateTime value) {
+      return new Filter(property, Operator.LESS_THAN, of(value), null);
+    }
+
+    public static Filter le(String property, Key value) {
+      return new Filter(property, Operator.LESS_THAN, of(value), null);
+    }
+
+    public static Filter le(String property, Blob value) {
+      return new Filter(property, Operator.LESS_THAN, of(value), null);
+    }
+
     public static Filter lte(String property, Value value) {
       return new Filter(property, Operator.LESS_THAN_OR_EQUAL, value, null);
     }
@@ -105,6 +141,10 @@ public static Filter hasAncestor(String property) {
       return new Filter(property, Operator.HAS_ANCESTOR, null, null);
     }
 
+    public static Filter isNull(String property) {
+      return new Filter(property, Operator.IS_NULL, null, null);
+    }
+
     public static Filter and(Filter first, Filter... other) {
       Set combined = new LinkedHashSet<>();
       for (Filter f : Iterables.concat(Collections.singleton(first), Arrays.asList(other))) {
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index a9fe6e53c005..a1cd276e4be7 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -21,8 +21,8 @@ public class DatastoreServiceTest {
   private static final String DATASET = "dataset1";
   private static final String KIND1 = "kind1";
   private static final String KIND2 = "kind2";
-  private static final NullValue NULL_VALUE = new NullValue();
-  private static final StringValue STR_VALUE = new StringValue("str");
+  private static final NullValue NULL_VALUE = NullValue.of();
+  private static final StringValue STR_VALUE = StringValue.of("str");
   private static final BooleanValue BOOL_VALUE = BooleanValue.builder(false).indexed(false).build();
   private static final PartialKey PARTIAL_KEY1 = PartialKey.builder(DATASET, KIND1).build();
   private static final PartialKey PARTIAL_KEY2 = PartialKey.builder(DATASET, KIND2).build();
@@ -31,18 +31,18 @@ public class DatastoreServiceTest {
   private static final Key KEY3 = Key.builder(KEY2).name("bla").build();
   private static final Key KEY4 = KEY2.newKey("newName1");
   private static final Key KEY5 = KEY2.newKey("newName2");
-  private static final KeyValue KEY_VALUE = new KeyValue(KEY1);
+  private static final KeyValue KEY_VALUE = KeyValue.of(KEY1);
   private static final ListValue LIST_VALUE1 = ListValue.builder()
       .addValue(NULL_VALUE)
       .addValue(STR_VALUE, BOOL_VALUE)
       .build();
-  private static final ListValue LIST_VALUE2 = new ListValue(Collections.singletonList(KEY_VALUE));
+  private static final ListValue LIST_VALUE2 = ListValue.of(Collections.singletonList(KEY_VALUE));
   private static final PartialEntity PARTIAL_ENTITY1 = PartialEntity.builder(PARTIAL_KEY2)
       .set("str", STR_VALUE).set("bool", BOOL_VALUE).set("list", LIST_VALUE1).build();
   private static final PartialEntity PARTIAL_ENTITY2 = PartialEntity.builder(PARTIAL_ENTITY1)
       .remove("str").set("bool", true).set("list", LIST_VALUE1.get()).build();
   private static final Entity ENTITY1 = Entity.builder(KEY1).set("str", STR_VALUE)
-      .set("bool", BOOL_VALUE).set("partial1", new EntityValue(PARTIAL_ENTITY1))
+      .set("bool", BOOL_VALUE).set("partial1", EntityValue.of(PARTIAL_ENTITY1))
       .set("list", LIST_VALUE2).build();
   private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
       .setNull("null").build();
@@ -143,7 +143,7 @@ public void testNewTransactionRollback() {
     Transaction transaction = datastore.newTransaction();
     transaction.add(ENTITY3);
     Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla")
-        .set("list3", new StringValue("bla"), StringValue.builder("bla").build()).build();
+        .set("list3", StringValue.of("bla"), StringValue.builder("bla").build()).build();
     transaction.update(entity2);
     transaction.delete(KEY1);
     transaction.rollback();
@@ -200,7 +200,7 @@ public void testNewBatchWriter() {
     BatchWriter batchWriter = datastore.newBatchWriter();
     Entity entity1 = Entity.builder(ENTITY1).clear().build();
     Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
-    Entity entity4 = Entity.builder(KEY4).set("value", new StringValue("value")).build();
+    Entity entity4 = Entity.builder(KEY4).set("value", StringValue.of("value")).build();
     Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
     batchWriter.add(entity4, entity5);
     batchWriter.put(ENTITY3, entity1, entity2);
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index eb63574ec021..ce3ea54f0a85 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -40,20 +40,19 @@ public class SerializationTest {
       .addArgument(20)
       .namespace("ns1")
       .build();
-  private static final KeyValue KEY_VALUE = new KeyValue(KEY1);
+  private static final KeyValue KEY_VALUE = KeyValue.of(KEY1);
   private static final NullValue NULL_VALUE = NullValue.builder().indexed(true).build();
-  private static final StringValue STRING_VALUE = new StringValue("hello");
-  private static final LongValue LONG_VALUE = new LongValue(123);
-  private static final DoubleValue DOUBLE_VALUE = new DoubleValue(12.34);
-  private static final BooleanValue BOOLEAN_VALUE = new BooleanValue(true);
-  private static final DateTimeValue DATE_AND_TIME_VALUE =
-      new DateTimeValue(DateTime.now());
-  private static final BlobValue BLOB_VALUE = new BlobValue(BLOB1);
-  private static final RawValue RAW_VALUE = new RawValue(
+  private static final StringValue STRING_VALUE = StringValue.of("hello");
+  private static final LongValue LONG_VALUE = LongValue.of(123);
+  private static final DoubleValue DOUBLE_VALUE = DoubleValue.of(12.34);
+  private static final BooleanValue BOOLEAN_VALUE = BooleanValue.of(true);
+  private static final DateTimeValue DATE_AND_TIME_VALUE = DateTimeValue.of(DateTime.now());
+  private static final BlobValue BLOB_VALUE = BlobValue.of(BLOB1);
+  private static final RawValue RAW_VALUE = RawValue.of(
       DatastoreV1.Value.newBuilder().setBlobKeyValue("blob-key").setMeaning(18).build());
   private static final Entity ENTITY1 = Entity.builder(KEY1).build();
   private static final Entity ENTITY2 =
-      Entity.builder(KEY2).set("null", new NullValue()).build();
+      Entity.builder(KEY2).set("null", NullValue.of()).build();
   private static final Entity ENTITY3 = Entity.builder(KEY2)
       .set("p1", StringValue.builder("hi1").meaning(10).build())
       .set("p2", StringValue.builder("hi2").meaning(11).indexed(false).build())
@@ -66,12 +65,9 @@ public class SerializationTest {
       .set("p1", STRING_VALUE)
       .set("p2", LongValue.builder(100).indexed(false).meaning(100).build())
       .build();
-  private static final EntityValue EMBEDDED_ENTITY_VALUE1 =
-      new EntityValue(EMBEDDED_ENTITY1);
-  private static final EntityValue EMBEDDED_ENTITY_VALUE2 =
-      new EntityValue(EMBEDDED_ENTITY2);
-  private static final EntityValue EMBEDDED_ENTITY_VALUE3 =
-      new EntityValue(EMBEDDED_ENTITY3);
+  private static final EntityValue EMBEDDED_ENTITY_VALUE1 = EntityValue.of(EMBEDDED_ENTITY1);
+  private static final EntityValue EMBEDDED_ENTITY_VALUE2 = EntityValue.of(EMBEDDED_ENTITY2);
+  private static final EntityValue EMBEDDED_ENTITY_VALUE3 = EntityValue.of(EMBEDDED_ENTITY3);
   private static final ListValue LIST_VALUE = ListValue.builder()
       .addValue(NULL_VALUE)
       .addValue(STRING_VALUE)

From 761fd07121353f2f9375ddb3f39a2b367b2c1fa1 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 12 Dec 2014 17:38:54 -0800
Subject: [PATCH 052/732] work in progress

---
 .classpath                                    |  10 +
 .settings/org.eclipse.jdt.core.prefs          |   3 +
 .../com/google/gcloud/datastore/BaseKey.java  |  11 +-
 .../gcloud/datastore/BatchWriterImpl.java     |   2 +-
 .../gcloud/datastore/DatastoreService.java    |   2 +-
 .../datastore/DatastoreServiceImpl.java       |  17 +-
 .../datastore/DatastoreServiceOptions.java    |  28 +-
 .../com/google/gcloud/datastore/GqlQuery.java |  41 ++-
 .../google/gcloud/datastore/KeyBuilder.java   |   4 +-
 .../com/google/gcloud/datastore/Query.java    |  15 +-
 .../gcloud/datastore/QueryResultImpl.java     |  42 ++-
 .../gcloud/datastore/StructuredQuery.java     | 295 ++++++++++++++----
 .../google/gcloud/datastore/Validator.java    |  44 +++
 .../datastore/DatastoreServiceTest.java       |   2 +-
 14 files changed, 357 insertions(+), 159 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/Validator.java

diff --git a/.classpath b/.classpath
index 9ed6ee4d0713..d6cf6121af66 100644
--- a/.classpath
+++ b/.classpath
@@ -23,5 +23,15 @@
 		
 	
 	
+	
+		
+			
+		
+	
+	
+		
+			
+		
+	
 	
 
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
index 0192f1f052aa..62f72418e536 100644
--- a/.settings/org.eclipse.jdt.core.prefs
+++ b/.settings/org.eclipse.jdt.core.prefs
@@ -8,6 +8,8 @@ org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonN
 org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
 org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
 org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.compliance=1.7
 org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
 org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
 org.eclipse.jdt.core.compiler.problem.comparingIdentical=error
@@ -90,3 +92,4 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning
 org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=warning
 org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning
 org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning
+org.eclipse.jdt.core.compiler.source=1.7
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index 4b4a33aa404e..2812574f6dbe 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -1,8 +1,9 @@
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.gcloud.datastore.DatastoreServiceOptions.validateDataset;
-import static com.google.gcloud.datastore.DatastoreServiceOptions.validateNamespace;
+import static com.google.gcloud.datastore.Validator.validateDataset;
+import static com.google.gcloud.datastore.Validator.validateKind;
+import static com.google.gcloud.datastore.Validator.validateNamespace;
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Preconditions;
@@ -84,12 +85,6 @@ public B kind(String kind) {
       return self();
     }
 
-    private String validateKind(String kind) {
-      checkArgument(!Strings.isNullOrEmpty(kind), "kind must not be empty or null");
-      checkArgument(kind.length() <= 500, "kind must not contain more than 500 characters");
-      return kind;
-    }
-
     public B clearPath() {
       ancestors.clear();
       return self();
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
index 0564d9b18c30..2395c2968c54 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
@@ -27,7 +27,7 @@ class BatchWriterImpl implements BatchWriter {
     if (optionsMap.containsKey(ForceWrites.class)) {
       force = ((ForceWrites) optionsMap.get(ForceWrites.class)).force();
     } else {
-      force = datastore.getOptions().force();
+      force = datastore.options().force();
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index b7633c49cd10..efd41bd9544e 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -10,7 +10,7 @@ public interface DatastoreService extends DatastoreReader, DatastoreWriter {
   /**
    * Returns the {@code DatastoreServiceOptions} for this service.
    */
-  DatastoreServiceOptions getOptions();
+  DatastoreServiceOptions options();
 
   /**
    * Returns a key builder for the requested {@code kind}.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 8fe92407260f..d55aaa47a380 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -28,7 +28,7 @@ final class DatastoreServiceImpl implements DatastoreService {
   }
 
   @Override
-  public DatastoreServiceOptions getOptions() {
+  public DatastoreServiceOptions options() {
     return options;
   }
 
@@ -53,20 +53,7 @@ public  QueryResult runQuery(Query query) {
   }
 
    QueryResult runQuery(DatastoreV1.ReadOptions readOptionsPb, Query query) {
-    DatastoreV1.RunQueryRequest.Builder requestPbBuilder = DatastoreV1.RunQueryRequest.newBuilder();
-    if (readOptionsPb != null) {
-      requestPbBuilder.setReadOptions(readOptionsPb);
-    }
-    DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder();
-    partitionIdPb.setDatasetId(options.dataset());
-    String namespace = query.namespace() != null ? query.namespace() : options.namespace();
-    if (namespace != null) {
-      partitionIdPb.setNamespace(namespace);
-    }
-    requestPbBuilder.setPartitionId(partitionIdPb.build());
-    query.populatePb(requestPbBuilder, 0, null);
-    DatastoreV1.RunQueryRequest requestPb = requestPbBuilder.build();
-    return new QueryResultImpl<>(this, query, requestPb, runQuery(requestPb).getBatch());
+    return new QueryResultImpl<>(this, readOptionsPb, query);
   }
 
   DatastoreV1.RunQueryResponse runQuery(DatastoreV1.RunQueryRequest requestPb) {
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 2f41df15a855..c0d5e3aee3a1 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -2,27 +2,21 @@
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.gcloud.datastore.Validator.validateDataset;
+import static com.google.gcloud.datastore.Validator.validateNamespace;
 
 import com.google.api.services.datastore.client.DatastoreOptions;
-import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
 import com.google.gcloud.ServiceOptions;
 
 import java.lang.reflect.Method;
 import java.util.Set;
-import java.util.regex.Pattern;
 
 public class DatastoreServiceOptions extends ServiceOptions {
 
   private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore";
   private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email";
   private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE);
-  private static final Pattern DATASET_PATTERN = Pattern.compile(
-      "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}\\:)?([a-z\\d][a-z\\d\\-]{0,99})");
-  private static final int MAX_NAMESPACE_LENGTH = 100;
-  private static final Pattern NAMESPACE_PATTERN =
-      Pattern.compile(String.format("[0-9A-Za-z\\._\\-]{0,%d}", MAX_NAMESPACE_LENGTH));
-
   private final String dataset;
   private final String namespace;
   private final boolean force;
@@ -79,24 +73,6 @@ public String namespace() {
     return namespace;
   }
 
-  static String validateDataset(String dataset) {
-    checkArgument(!Strings.isNullOrEmpty(dataset), "dataset can't be empty or null");
-    checkArgument(DATASET_PATTERN.matcher(dataset).matches(),
-          "dataset must match the following pattern: " + DATASET_PATTERN.pattern());
-    return dataset;
-  }
-
-  static String validateNamespace(String namespace) {
-    if (namespace != null) {
-      checkArgument(!namespace.isEmpty(), "namespace must not be an empty string");
-      checkArgument(namespace.length() <= 100,
-          "namespace must not contain more than 100 characters");
-      checkArgument(NAMESPACE_PATTERN.matcher(namespace).matches(),
-          "namespace must the following pattern: " + NAMESPACE_PATTERN.pattern());
-    }
-    return namespace;
-  }
-
   private static String defaultNamespace() {
     // TODO(ozarov): An alternative to reflection would be to depend on AE api jar:
     // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index d7c481d613ee..a1ffa9f6ff70 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -1,6 +1,7 @@
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.gcloud.datastore.Validator.validateNamespace;
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.MoreObjects;
@@ -10,7 +11,6 @@
 import com.google.common.primitives.Booleans;
 import com.google.common.primitives.Doubles;
 import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.util.ArrayList;
@@ -134,7 +134,7 @@ public Builder query(String query) {
     }
 
     public Builder namespace(String namespace) {
-      this.namespace = namespace;
+      this.namespace = validateNamespace(namespace);
       return this;
     }
 
@@ -239,15 +239,15 @@ public Builder addArgument(Blob... value) {
       return this;
     }
 
+    public GqlQuery build() {
+      return new GqlQuery<>(this);
+    }
+
     @SuppressWarnings("rawtypes")
     private static Argument toArgument(Value.BuilderFactory builderFactory, List values) {
       return toArgument(null, builderFactory, values);
     }
 
-    public GqlQuery build() {
-      return new GqlQuery<>(this);
-    }
-
     @SuppressWarnings({"unchecked", "rawtypes"})
     private static Argument toArgument(String name, Value.BuilderFactory builderFactory,
         List values) {
@@ -341,14 +341,29 @@ protected DatastoreV1.GqlQuery toPb() {
   }
 
   @Override
-  protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead,
-      ByteString batchCursor) {
-    if (batchCursor == null) {
-      requestPb.setGqlQuery(toPb());
-      return;
-    }
-    // see b/18705483
+  protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
+    requestPb.setGqlQuery(toPb());
+  }
+
+  @Override
+  protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
     throw new UnsupportedOperationException("paging not implemented yet");
+    /*
+    // TODO: THIS IS A MAJOR HACK, remove when possible. see b/18705483
+    String PREFIX_GROUP = "\\s*(?SELECT .*?)";
+    String OFFSET_GROUP =
+        "(\\s+OFFSET\\s+(?[^\\s]+)(\\s+\\+\\s+(?[^\\s]+))?)?";
+    String LIMIT_GROUP =
+        "(\\s+LIMIT\\s+((?[^\\s]+)|FIRST \\((?[^,]+,[^\\)]+)\\)))?\\s*";
+    Pattern pattern =
+        Pattern.compile(PREFIX_GROUP + OFFSET_GROUP + LIMIT_GROUP, Pattern.CASE_INSENSITIVE);
+
+    Matcher matcher = pattern.matcher(queryString);
+
+    if (!matcher.matches()) {
+      throw new UnsupportedOperationException("paging for this query is not implemented yet");
+    }
+    */
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java
index 6860063124d0..a906a29d423c 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java
@@ -16,9 +16,9 @@ public final class KeyBuilder extends BaseKey.Builder {
    * Constructing a KeyBuilder.
    */
   public KeyBuilder(DatastoreService service, String kind) {
-    super(checkNotNull(service).getOptions().dataset(), kind);
+    super(checkNotNull(service).options().dataset(), kind);
     this.service = service;
-    namespace(service.getOptions().namespace());
+    namespace(service.options().namespace());
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index d1c29d189599..bf474bc73d1e 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -1,10 +1,11 @@
 package com.google.gcloud.datastore;
 
+import static com.google.api.client.util.Preconditions.checkNotNull;
+
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
 import com.google.gcloud.datastore.QueryResult.Type;
-import com.google.protobuf.ByteString;
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 
@@ -67,7 +68,7 @@ public static ResultType keyOnly() {
   }
 
   Query(ResultType resultType, String namespace) {
-    this.resultType = resultType;
+    this.resultType = checkNotNull(resultType);
     this.namespace = namespace;
   }
 
@@ -95,13 +96,7 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   protected abstract Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException;
 
-  protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead,
-      ByteString batchCursor);
+  protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb);
 
-  /**
-   * Returns a new structured query builder.
-   */
-  public static StructuredQuery.Builder builder() {
-    return new StructuredQuery.Builder();
-  }
+  protected abstract Query nextQuery(DatastoreV1.QueryResultBatch responsePb);
 }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index f24130212ab8..734b8b1bee4a 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -13,9 +13,10 @@ class QueryResultImpl implements QueryResult {
       RESULT_TYPE_CONVERTER;
 
   private final DatastoreServiceImpl datastore;
-  private final Query query;
+  private final DatastoreV1.ReadOptions readOptionsPb;
   private final QueryResult.Type type;
-  private DatastoreV1.RunQueryRequest requestPb;
+  private Query query;
+  private DatastoreV1.QueryResultBatch resultPb;
   private Iterator entityResultPbIter;
   private ByteString endCursor;
   private int count;
@@ -29,37 +30,52 @@ class QueryResultImpl implements QueryResult {
     RESULT_TYPE_CONVERTER = builder.build();
   }
 
-  QueryResultImpl(DatastoreServiceImpl datastore, Query query,
-      DatastoreV1.RunQueryRequest requestPb, DatastoreV1.QueryResultBatch resultPb) {
+  QueryResultImpl(DatastoreServiceImpl datastore, DatastoreV1.ReadOptions readOptionsPb,
+      Query query) {
     this.datastore = datastore;
+    this.readOptionsPb = readOptionsPb;
     this.query = query;
-    this.requestPb = requestPb;
+    sendRequest();
     type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
     Preconditions.checkState(query.resultType().getType() == null
         || query.resultType().getType() == type, "Unexpected result type");
   }
 
-  void setQueryResultBatch(DatastoreV1.QueryResultBatch resultPb) {
+  private DatastoreV1.QueryResultBatch sendRequest() {
+    DatastoreV1.RunQueryRequest.Builder requestPb = DatastoreV1.RunQueryRequest.newBuilder();
+    if (readOptionsPb != null) {
+      requestPb.setReadOptions(readOptionsPb);
+    }
+    DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder();
+    partitionIdPb.setDatasetId(datastore.options().dataset());
+    String namespace = query.namespace() != null
+        ? query.namespace()
+        : datastore.options().namespace();
+    if (namespace != null) {
+      partitionIdPb.setNamespace(namespace);
+    }
+    requestPb.setPartitionId(partitionIdPb.build());
+    query.populatePb(requestPb);
+    resultPb = datastore.runQuery(requestPb.build()).getBatch();
     entityResultPbIter = resultPb.getEntityResultList().iterator();
     if (DatastoreV1.QueryResultBatch.MoreResultsType.NOT_FINISHED == resultPb.getMoreResults()) {
       endCursor = resultPb.getEndCursor();
+    } else {
+      endCursor = null;
     }
+    return resultPb;
   }
 
   @Override
   public boolean hasNext() {
-    return entityResultPbIter.hasNext()  || endCursor != null;
+    return entityResultPbIter.hasNext() || endCursor != null;
   }
 
   @Override
   public T next() {
     if (!hasNext() && endCursor != null) {
-      DatastoreV1.RunQueryRequest.Builder requestPbBuilder = requestPb.toBuilder();
-      query.populatePb(requestPbBuilder, count, endCursor);
-      DatastoreV1.RunQueryRequest newRequestPb = requestPbBuilder.build();
-      DatastoreV1.RunQueryResponse responsePb = datastore.runQuery(newRequestPb);
-      requestPb = newRequestPb;
-      setQueryResultBatch(responsePb.getBatch());
+      query = query.nextQuery(resultPb);
+      sendRequest();
     }
     DatastoreV1.Entity entity = entityResultPbIter.next().getEntity();
     count++;
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 77ee46e1438b..8c172b717051 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -12,29 +12,26 @@
 import com.google.api.client.util.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
-import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.io.Serializable;
 import java.util.ArrayDeque;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
-public final class StructuredQuery extends Query {
+public class StructuredQuery extends Query {
 
   private static final long serialVersionUID = 546838955624019594L;
 
   private final transient boolean keysOnly;
   private final transient String kind;
-  private final transient ImmutableList projection;
   private final transient Filter filter;
-  private final transient ImmutableList groupBy;
   private final transient ImmutableList orderBy;
   private final transient Cursor startCursor;
   private final transient Cursor endCursor;
@@ -42,7 +39,33 @@ public final class StructuredQuery extends Query {
   private final transient Integer limit;
 
 
-  public static final class Filter implements Serializable {
+  public static class Expression implements Serializable {
+
+    private static final long serialVersionUID = -6443285436239990860L;
+
+    private final Operator operator = null;
+    private final List values = null;
+
+    enum Operator {
+      AND,
+      OR
+    }
+
+    /*
+    Expression(Operator operator, Expression first, Expression... other) {
+      this.operator = operator;
+      ImmutableList.builder().
+      this.
+    }
+    */
+  }
+
+/*
+  public static final class And extends Expression {
+
+  }
+*/
+  public static final class Filter extends Expression {
 
     private static final long serialVersionUID = -4514695915258598597L;
 
@@ -51,7 +74,7 @@ public static final class Filter implements Serializable {
     private final Value value;
     private final Filter next;
 
-    public enum Operator {
+    enum Operator {
       LESS_THAN,
       LESS_THAN_OR_EQUAL,
       GREATER_THAN,
@@ -62,10 +85,24 @@ public enum Operator {
       AND
     }
 
-    private Filter(String property, Operator operator, Value value, Filter next) {
+    private Filter(String property, Operator operator, Value value) {
+      this.property = checkNotNull(property);
+      this.operator = checkNotNull(operator);
+      this.value = checkNotNull(value);
+      this.next = null;
+    }
+
+    private Filter(String property, Operator operator) {
       this.property = checkNotNull(property);
       this.operator = checkNotNull(operator);
-      this.value = value;
+      this.value = null;
+      this.next = null;
+    }
+
+    private Filter(Filter from, Filter next) {
+      this.property = from.property;
+      this.operator = from.operator;
+      this.value = from.value;
       this.next = next;
     }
 
@@ -90,66 +127,178 @@ public boolean equals(Object obj) {
     }
 
     public static Filter le(String property, Value value) {
-      return new Filter(property, Operator.LESS_THAN, value, null);
+      return new Filter(property, Operator.LESS_THAN, value);
     }
 
     public static Filter le(String property, String value) {
-      return new Filter(property, Operator.LESS_THAN, of(value), null);
+      return new Filter(property, Operator.LESS_THAN, of(value));
     }
 
     public static Filter le(String property, long value) {
-      return new Filter(property, Operator.LESS_THAN, of(value), null);
+      return new Filter(property, Operator.LESS_THAN, of(value));
     }
 
     public static Filter le(String property, double value) {
-      return new Filter(property, Operator.LESS_THAN, of(value), null);
+      return new Filter(property, Operator.LESS_THAN, of(value));
     }
 
     public static Filter le(String property, boolean value) {
-      return new Filter(property, Operator.LESS_THAN, of(value), null);
+      return new Filter(property, Operator.LESS_THAN, of(value));
     }
 
     public static Filter le(String property, DateTime value) {
-      return new Filter(property, Operator.LESS_THAN, of(value), null);
+      return new Filter(property, Operator.LESS_THAN, of(value));
     }
 
     public static Filter le(String property, Key value) {
-      return new Filter(property, Operator.LESS_THAN, of(value), null);
+      return new Filter(property, Operator.LESS_THAN, of(value));
     }
 
     public static Filter le(String property, Blob value) {
-      return new Filter(property, Operator.LESS_THAN, of(value), null);
+      return new Filter(property, Operator.LESS_THAN, of(value));
     }
 
     public static Filter lte(String property, Value value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, value, null);
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, value);
+    }
+
+    public static Filter lte(String property, String value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter lte(String property, long value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter lte(String property, double value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter lte(String property, boolean value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter lte(String property, DateTime value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter lte(String property, Key value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter lte(String property, Blob value) {
+      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
     public static Filter gt(String property, Value value) {
-      return new Filter(property, Operator.GREATER_THAN, value, null);
+      return new Filter(property, Operator.GREATER_THAN, value);
+    }
+
+    public static Filter gt(String property, String value) {
+      return new Filter(property, Operator.GREATER_THAN, of(value));
+    }
+
+    public static Filter gt(String property, long value) {
+      return new Filter(property, Operator.GREATER_THAN, of(value));
+    }
+
+    public static Filter gt(String property, double value) {
+      return new Filter(property, Operator.GREATER_THAN, of(value));
+    }
+
+    public static Filter gt(String property, boolean value) {
+      return new Filter(property, Operator.GREATER_THAN, of(value));
+    }
+
+    public static Filter gt(String property, DateTime value) {
+      return new Filter(property, Operator.GREATER_THAN, of(value));
+    }
+
+    public static Filter gt(String property, Key value) {
+      return new Filter(property, Operator.GREATER_THAN, of(value));
+    }
+
+    public static Filter gt(String property, Blob value) {
+      return new Filter(property, Operator.GREATER_THAN, of(value));
     }
 
     public static Filter gte(String property, Value value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, value, null);
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, value);
+    }
+
+    public static Filter gte(String property, String value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter gte(String property, long value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter gte(String property, double value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter gte(String property, boolean value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter gte(String property, DateTime value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter gte(String property, Key value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    }
+
+    public static Filter gte(String property, Blob value) {
+      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
     public static Filter eq(String property, Value value) {
-      return new Filter(property, Operator.EQUAL, value, null);
+      return new Filter(property, Operator.EQUAL, value);
+    }
+
+    public static Filter eq(String property, String value) {
+      return new Filter(property, Operator.EQUAL, of(value));
+    }
+
+    public static Filter eq(String property, long value) {
+      return new Filter(property, Operator.EQUAL, of(value));
+    }
+
+    public static Filter eq(String property, double value) {
+      return new Filter(property, Operator.EQUAL, of(value));
+    }
+
+    public static Filter eq(String property, boolean value) {
+      return new Filter(property, Operator.EQUAL, of(value));
+    }
+
+    public static Filter eq(String property, DateTime value) {
+      return new Filter(property, Operator.EQUAL, of(value));
+    }
+
+    public static Filter eq(String property, Key value) {
+      return new Filter(property, Operator.EQUAL, of(value));
+    }
+
+    public static Filter eq(String property, Blob value) {
+      return new Filter(property, Operator.EQUAL, of(value));
     }
 
     public static Filter hasAncestor(String property) {
-      return new Filter(property, Operator.HAS_ANCESTOR, null, null);
+      return new Filter(property, Operator.HAS_ANCESTOR);
     }
 
     public static Filter isNull(String property) {
-      return new Filter(property, Operator.IS_NULL, null, null);
+      return new Filter(property, Operator.IS_NULL);
     }
 
     public static Filter and(Filter first, Filter... other) {
       Set combined = new LinkedHashSet<>();
       for (Filter f : Iterables.concat(Collections.singleton(first), Arrays.asList(other))) {
         while (f != null) {
-          combined.add(new Filter(f.property, f.operator, f.value, null));
+          combined.add(new Filter(f, null));
           f = f.next;
         }
       }
@@ -158,12 +307,12 @@ public static Filter and(Filter first, Filter... other) {
         stack.push(f);
       }
       while (true) {
-        Filter f1 = stack.pop();
+        Filter top = stack.pop();
         if (stack.isEmpty()) {
-          return f1;
+          return top;
         }
-        Filter f2 = stack.pop();
-        stack.push(new Filter(f2.property, f2.operator, f2.value, f1));
+        Filter beforeTop = stack.pop();
+        stack.push(new Filter(beforeTop, top));
       }
     }
   }
@@ -219,7 +368,7 @@ public static OrderBy desc(String property) {
     }
   }
 
-  static class Builder {
+  static class Builder> {
 
     private String namespace;
     private String kind;
@@ -227,62 +376,68 @@ static class Builder {
     private Cursor endCursor;
     private Integer offset;
     private Integer limit;
+    private Filter filter;
+    private List orderBy = new LinkedList<>();
 
-    public Builder namespace(String namespace) {
+    @SuppressWarnings("unchecked")
+    protected B self() {
+      return (B) this;
+    }
+
+    public B namespace(String namespace) {
       this.namespace = namespace;
-      return this;
+      return self();
     }
 
-    public Builder kind(String kind) {
+    public B kind(String kind) {
       this.kind = kind;
-      return this;
+      return self();
     }
 
-    public Builder startCursor(Cursor startCursor) {
+    public B startCursor(Cursor startCursor) {
       this.startCursor = startCursor;
-      return this;
+      return self();
     }
 
-    public Builder encCursor(Cursor endCursor) {
+    public B encCursor(Cursor endCursor) {
       this.endCursor = endCursor;
-      return this;
+      return self();
     }
 
-    public Builder offset(Integer offset) {
+    public B offset(Integer offset) {
       Preconditions.checkArgument(offset == null || offset >= 0, "offset must not be negative");
       this.offset = offset;
-      return this;
+      return self();
     }
 
-    public Builder limit(Integer limit) {
+    public B limit(Integer limit) {
       Preconditions.checkArgument(limit == null || offset > 0, "limit must be positive");
       this.limit = limit;
-      return this;
+      return self();
     }
 
-    /*
-    public StructuredQuery full() {
-      return new StructuredQuery<>(ResultType.full(), this, false);
+    public B filter(Filter filter) {
+      this.filter = filter;
+      return self();
     }
 
-    public StructuredQuery projection(String projection, String... other) {
-      ImmutableSet projections =
-          ImmutableSet.builder().add(projection).add(other).build();
-      return new StructuredQuery<>(ResultType.projection(), this, projections);
+    public B clearOrderBy() {
+      orderBy.clear();
+      return self();
     }
 
-    public StructuredQuery keyOnly() {
-      return new StructuredQuery<>(ResultType.keyOnly(), this, true);
+    public B addOrderBy(OrderBy orderBy, OrderBy... others) {
+      this.orderBy.add(orderBy);
+      for (OrderBy other : others) {
+        this.orderBy.add(other);
+      }
+      return self();
     }
-    */
   }
 
-
-
   private StructuredQuery(ResultType resultType, boolean keysOnly, String namespace, String kind,
       Cursor startCursor, Cursor endCursor, Integer offset, Integer limit,
-      ImmutableList projection, Filter filter, ImmutableList groupBy,
-      ImmutableList orderBy) {
+      Filter filter, ImmutableList orderBy) {
     super(resultType, namespace);
     this.keysOnly = keysOnly;
     this.kind = kind;
@@ -290,17 +445,14 @@ private StructuredQuery(ResultType resultType, boolean keysOnly, String names
     this.endCursor = endCursor;
     this.offset = offset;
     this.limit = limit;
-    this.projection = projection;
     this.filter = filter;
-    this.groupBy = groupBy;
     this.orderBy = orderBy;
-    Preconditions.checkState(keysOnly == false || projection.isEmpty() && groupBy.isEmpty(),
-        "projection or group by are not applicable for keys-only queries");
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(namespace(), keysOnly, kind, startCursor, endCursor, offset, limit, projection, filter, groupBy, orderBy);
+    return Objects.hash(
+        namespace(), keysOnly, kind, startCursor, endCursor, offset, limit, filter, orderBy);
   }
 
   @Override
@@ -318,19 +470,18 @@ public boolean equals(Object obj) {
         && Objects.equals(endCursor, other.endCursor)
         && Objects.equals(offset, other.offset)
         && Objects.equals(limit, other.limit)
-        && Objects.equals(projection, other.projection)
         && Objects.equals(filter, other.filter)
-        && Objects.equals(groupBy, other.groupBy)
         && Objects.equals(orderBy, other.orderBy);
   }
 
   @Override
-  protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb, int totalRead,
-      ByteString batchCursor) {
-    if (batchCursor == null) {
-      requestPb.setQuery(toPb());
-      return;
-    }
+  protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
+    requestPb.setQuery(toPb());
+  }
+
+  @Override
+  protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
+    // TODO: implment
     throw new UnsupportedOperationException("paging not implemented yet");
   }
 
@@ -358,6 +509,12 @@ protected DatastoreV1.Query toPb() {
     if (limit != null) {
       queryPb.setLimit(limit);
     }
+    if (filter != null) {
+      DatastoreV1.Filter.Builder filterPb = queryPb.getFilterBuilder();
+      if (filter.next == null) {
+        // TODO
+      }
+    }
     // TODO: projection, filter, groupBy, orderBy (or keys-only)
     return queryPb.build();
   }
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/src/main/java/com/google/gcloud/datastore/Validator.java
new file mode 100644
index 000000000000..2f915ccc3d93
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/Validator.java
@@ -0,0 +1,44 @@
+package com.google.gcloud.datastore;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.base.Strings;
+
+import java.util.regex.Pattern;
+
+/**
+ * Utility to validate Datastore type/values.
+ */
+class Validator {
+
+  private static final Pattern DATASET_PATTERN = Pattern.compile(
+      "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}\\:)?([a-z\\d][a-z\\d\\-]{0,99})");
+  private static final int MAX_NAMESPACE_LENGTH = 100;
+  private static final Pattern NAMESPACE_PATTERN =
+      Pattern.compile(String.format("[0-9A-Za-z\\._\\-]{0,%d}", MAX_NAMESPACE_LENGTH));
+
+
+  static String validateDataset(String dataset) {
+    checkArgument(!Strings.isNullOrEmpty(dataset), "dataset can't be empty or null");
+    checkArgument(Validator.DATASET_PATTERN.matcher(dataset).matches(),
+          "dataset must match the following pattern: " + Validator.DATASET_PATTERN.pattern());
+    return dataset;
+  }
+
+  static String validateNamespace(String namespace) {
+    if (namespace != null) {
+      checkArgument(!namespace.isEmpty(), "namespace must not be an empty string");
+      checkArgument(namespace.length() <= 100,
+          "namespace must not contain more than 100 characters");
+      checkArgument(Validator.NAMESPACE_PATTERN.matcher(namespace).matches(),
+          "namespace must the following pattern: " + Validator.NAMESPACE_PATTERN.pattern());
+    }
+    return namespace;
+  }
+
+  static String validateKind(String kind) {
+    checkArgument(!Strings.isNullOrEmpty(kind), "kind must not be empty or null");
+    checkArgument(kind.length() <= 500, "kind must not contain more than 500 characters");
+    return kind;
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index a1cd276e4be7..60525f39f894 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -73,7 +73,7 @@ public void setUp() {
 
   @Test
   public void testGetOptions() {
-    assertSame(options, datastore.getOptions());
+    assertSame(options, datastore.options());
   }
 
   @Test

From 63bafd931fd828b8a11d903da8681c4844679394 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Fri, 12 Dec 2014 23:57:49 -0800
Subject: [PATCH 053/732] change filter composition

---
 .classpath                                    |  10 -
 .../gcloud/datastore/StructuredQuery.java     | 294 +++++++++---------
 2 files changed, 147 insertions(+), 157 deletions(-)

diff --git a/.classpath b/.classpath
index d6cf6121af66..9ed6ee4d0713 100644
--- a/.classpath
+++ b/.classpath
@@ -23,15 +23,5 @@
 		
 	
 	
-	
-		
-			
-		
-	
-	
-		
-			
-		
-	
 	
 
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 8c172b717051..684d31a8fcf4 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -12,18 +12,13 @@
 import com.google.api.client.util.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.io.Serializable;
-import java.util.ArrayDeque;
 import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Objects;
-import java.util.Set;
 
 public class StructuredQuery extends Query {
 
@@ -39,40 +34,60 @@ public class StructuredQuery extends Query {
   private final transient Integer limit;
 
 
-  public static class Expression implements Serializable {
+  public abstract static class Filter implements Serializable {
 
     private static final long serialVersionUID = -6443285436239990860L;
 
-    private final Operator operator = null;
-    private final List values = null;
+    Filter() {
+    }
+
+    protected abstract DatastoreV1.Filter toPb();
+  }
+
+
+  public static final class Expression extends Filter {
+
+    private static final long serialVersionUID = 3610352685739360009L;
+    private final Operator operator;
+    private final ImmutableList filters;
 
     enum Operator {
-      AND,
-      OR
+      AND;
+
+      DatastoreV1.CompositeFilter.Operator toPb() {
+        return DatastoreV1.CompositeFilter.Operator.valueOf(name());
+      }
     }
 
-    /*
-    Expression(Operator operator, Expression first, Expression... other) {
+    private Expression(Operator operator, Filter first, Filter... other) {
       this.operator = operator;
-      ImmutableList.builder().
-      this.
+      this.filters =
+          ImmutableList.builder().add(first).addAll(Arrays.asList(other)).build();
     }
-    */
-  }
 
-/*
-  public static final class And extends Expression {
+    public static Expression and(Filter first, Filter... other) {
+      return new Expression(Operator.AND, first, other);
+    }
 
+    @Override
+    protected DatastoreV1.Filter toPb() {
+      DatastoreV1.Filter.Builder filterPb = DatastoreV1.Filter.newBuilder();
+      DatastoreV1.CompositeFilter.Builder compositeFilterPb = filterPb.getCompositeFilterBuilder();
+      compositeFilterPb.setOperator(operator.toPb());
+      for (Filter filter : filters) {
+        compositeFilterPb.addFilter(filter.toPb());
+      }
+      return filterPb.build();
+    }
   }
-*/
-  public static final class Filter extends Expression {
+
+  public static final class Condition extends Filter {
 
     private static final long serialVersionUID = -4514695915258598597L;
 
     private final String property;
     private final Operator operator;
     private final Value value;
-    private final Filter next;
 
     enum Operator {
       LESS_THAN,
@@ -81,34 +96,33 @@ enum Operator {
       GREATER_THAN_OR_EQUAL,
       EQUAL,
       HAS_ANCESTOR,
-      IS_NULL,
-      AND
+      IS_NULL {
+        @Override
+        public DatastoreV1.PropertyFilter.Operator toPb() {
+          return DatastoreV1.PropertyFilter.Operator.valueOf(EQUAL.name());
+        }
+      };
+
+      public DatastoreV1.PropertyFilter.Operator toPb() {
+        return DatastoreV1.PropertyFilter.Operator.valueOf(name());
+      }
     }
 
-    private Filter(String property, Operator operator, Value value) {
+    private Condition(String property, Operator operator, Value value) {
       this.property = checkNotNull(property);
       this.operator = checkNotNull(operator);
       this.value = checkNotNull(value);
-      this.next = null;
     }
 
-    private Filter(String property, Operator operator) {
+    private Condition(String property, Operator operator) {
       this.property = checkNotNull(property);
       this.operator = checkNotNull(operator);
       this.value = null;
-      this.next = null;
-    }
-
-    private Filter(Filter from, Filter next) {
-      this.property = from.property;
-      this.operator = from.operator;
-      this.value = from.value;
-      this.next = next;
     }
 
     @Override
     public int hashCode() {
-      return Objects.hash(property, operator, value, next);
+      return Objects.hash(property, operator, value);
     }
 
     @Override
@@ -116,204 +130,193 @@ public boolean equals(Object obj) {
       if (obj == this) {
         return true;
       }
-      if (!(obj instanceof Filter)) {
+      if (!(obj instanceof Condition)) {
         return false;
       }
-      Filter other = (Filter) obj;
+      Condition other = (Condition) obj;
       return property.equals(other.property)
           && operator.equals(other.operator)
-          && Objects.equals(value, other.value)
-          && Objects.equals(next, other.next);
+          && Objects.equals(value, other.value);
     }
 
-    public static Filter le(String property, Value value) {
-      return new Filter(property, Operator.LESS_THAN, value);
+    public static Condition le(String property, Value value) {
+      return new Condition(property, Operator.LESS_THAN, value);
     }
 
-    public static Filter le(String property, String value) {
-      return new Filter(property, Operator.LESS_THAN, of(value));
+    public static Condition le(String property, String value) {
+      return new Condition(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Filter le(String property, long value) {
-      return new Filter(property, Operator.LESS_THAN, of(value));
+    public static Condition le(String property, long value) {
+      return new Condition(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Filter le(String property, double value) {
-      return new Filter(property, Operator.LESS_THAN, of(value));
+    public static Condition le(String property, double value) {
+      return new Condition(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Filter le(String property, boolean value) {
-      return new Filter(property, Operator.LESS_THAN, of(value));
+    public static Condition le(String property, boolean value) {
+      return new Condition(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Filter le(String property, DateTime value) {
-      return new Filter(property, Operator.LESS_THAN, of(value));
+    public static Condition le(String property, DateTime value) {
+      return new Condition(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Filter le(String property, Key value) {
-      return new Filter(property, Operator.LESS_THAN, of(value));
+    public static Condition le(String property, Key value) {
+      return new Condition(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Filter le(String property, Blob value) {
-      return new Filter(property, Operator.LESS_THAN, of(value));
+    public static Condition le(String property, Blob value) {
+      return new Condition(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Filter lte(String property, Value value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, value);
+    public static Condition lte(String property, Value value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, value);
     }
 
-    public static Filter lte(String property, String value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static Condition lte(String property, String value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter lte(String property, long value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static Condition lte(String property, long value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter lte(String property, double value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static Condition lte(String property, double value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter lte(String property, boolean value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static Condition lte(String property, boolean value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter lte(String property, DateTime value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static Condition lte(String property, DateTime value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter lte(String property, Key value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static Condition lte(String property, Key value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter lte(String property, Blob value) {
-      return new Filter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static Condition lte(String property, Blob value) {
+      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter gt(String property, Value value) {
-      return new Filter(property, Operator.GREATER_THAN, value);
+    public static Condition gt(String property, Value value) {
+      return new Condition(property, Operator.GREATER_THAN, value);
     }
 
-    public static Filter gt(String property, String value) {
-      return new Filter(property, Operator.GREATER_THAN, of(value));
+    public static Condition gt(String property, String value) {
+      return new Condition(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Filter gt(String property, long value) {
-      return new Filter(property, Operator.GREATER_THAN, of(value));
+    public static Condition gt(String property, long value) {
+      return new Condition(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Filter gt(String property, double value) {
-      return new Filter(property, Operator.GREATER_THAN, of(value));
+    public static Condition gt(String property, double value) {
+      return new Condition(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Filter gt(String property, boolean value) {
-      return new Filter(property, Operator.GREATER_THAN, of(value));
+    public static Condition gt(String property, boolean value) {
+      return new Condition(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Filter gt(String property, DateTime value) {
-      return new Filter(property, Operator.GREATER_THAN, of(value));
+    public static Condition gt(String property, DateTime value) {
+      return new Condition(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Filter gt(String property, Key value) {
-      return new Filter(property, Operator.GREATER_THAN, of(value));
+    public static Condition gt(String property, Key value) {
+      return new Condition(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Filter gt(String property, Blob value) {
-      return new Filter(property, Operator.GREATER_THAN, of(value));
+    public static Condition gt(String property, Blob value) {
+      return new Condition(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Filter gte(String property, Value value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, value);
+    public static Condition gte(String property, Value value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, value);
     }
 
-    public static Filter gte(String property, String value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static Condition gte(String property, String value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter gte(String property, long value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static Condition gte(String property, long value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter gte(String property, double value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static Condition gte(String property, double value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter gte(String property, boolean value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static Condition gte(String property, boolean value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter gte(String property, DateTime value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static Condition gte(String property, DateTime value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter gte(String property, Key value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static Condition gte(String property, Key value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter gte(String property, Blob value) {
-      return new Filter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static Condition gte(String property, Blob value) {
+      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Filter eq(String property, Value value) {
-      return new Filter(property, Operator.EQUAL, value);
+    public static Condition eq(String property, Value value) {
+      return new Condition(property, Operator.EQUAL, value);
     }
 
-    public static Filter eq(String property, String value) {
-      return new Filter(property, Operator.EQUAL, of(value));
+    public static Condition eq(String property, String value) {
+      return new Condition(property, Operator.EQUAL, of(value));
     }
 
-    public static Filter eq(String property, long value) {
-      return new Filter(property, Operator.EQUAL, of(value));
+    public static Condition eq(String property, long value) {
+      return new Condition(property, Operator.EQUAL, of(value));
     }
 
-    public static Filter eq(String property, double value) {
-      return new Filter(property, Operator.EQUAL, of(value));
+    public static Condition eq(String property, double value) {
+      return new Condition(property, Operator.EQUAL, of(value));
     }
 
-    public static Filter eq(String property, boolean value) {
-      return new Filter(property, Operator.EQUAL, of(value));
+    public static Condition eq(String property, boolean value) {
+      return new Condition(property, Operator.EQUAL, of(value));
     }
 
-    public static Filter eq(String property, DateTime value) {
-      return new Filter(property, Operator.EQUAL, of(value));
+    public static Condition eq(String property, DateTime value) {
+      return new Condition(property, Operator.EQUAL, of(value));
     }
 
-    public static Filter eq(String property, Key value) {
-      return new Filter(property, Operator.EQUAL, of(value));
+    public static Condition eq(String property, Key value) {
+      return new Condition(property, Operator.EQUAL, of(value));
     }
 
-    public static Filter eq(String property, Blob value) {
-      return new Filter(property, Operator.EQUAL, of(value));
+    public static Condition eq(String property, Blob value) {
+      return new Condition(property, Operator.EQUAL, of(value));
     }
 
-    public static Filter hasAncestor(String property) {
-      return new Filter(property, Operator.HAS_ANCESTOR);
+    public static Condition hasAncestor(String property) {
+      return new Condition(property, Operator.HAS_ANCESTOR);
     }
 
-    public static Filter isNull(String property) {
-      return new Filter(property, Operator.IS_NULL);
+    public static Condition isNull(String property) {
+      return new Condition(property, Operator.IS_NULL, NullValue.of());
     }
 
-    public static Filter and(Filter first, Filter... other) {
-      Set combined = new LinkedHashSet<>();
-      for (Filter f : Iterables.concat(Collections.singleton(first), Arrays.asList(other))) {
-        while (f != null) {
-          combined.add(new Filter(f, null));
-          f = f.next;
-        }
-      }
-      ArrayDeque stack = new ArrayDeque<>(combined.size());
-      for (Filter f : combined) {
-        stack.push(f);
-      }
-      while (true) {
-        Filter top = stack.pop();
-        if (stack.isEmpty()) {
-          return top;
-        }
-        Filter beforeTop = stack.pop();
-        stack.push(new Filter(beforeTop, top));
+    @Override
+    protected DatastoreV1.Filter toPb() {
+      DatastoreV1.Filter.Builder filterPb = DatastoreV1.Filter.newBuilder();
+      DatastoreV1.PropertyFilter.Builder propertyFilterPb = filterPb.getPropertyFilterBuilder();
+      propertyFilterPb.getPropertyBuilder().setName(property);
+      propertyFilterPb.setOperator(operator.toPb());
+      if (value != null) {
+        propertyFilterPb.setValue(value.toPb());
       }
+      return filterPb.build();
     }
   }
 
@@ -510,12 +513,9 @@ protected DatastoreV1.Query toPb() {
       queryPb.setLimit(limit);
     }
     if (filter != null) {
-      DatastoreV1.Filter.Builder filterPb = queryPb.getFilterBuilder();
-      if (filter.next == null) {
-        // TODO
-      }
+      queryPb.setFilter(filter.toPb());
     }
-    // TODO: projection, filter, groupBy, orderBy (or keys-only)
+    // TODO: projection, groupBy, orderBy (or keys-only)
     return queryPb.build();
   }
 

From a18313fbf6c8816c8e0cfadcfd94a56942449f35 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 14 Dec 2014 11:53:21 -0800
Subject: [PATCH 054/732] structured query - work in progress

---
 .../com/google/gcloud/datastore/GqlQuery.java |  18 +-
 .../com/google/gcloud/datastore/Query.java    | 130 +++--
 .../google/gcloud/datastore/QueryResult.java  |  43 +-
 .../gcloud/datastore/QueryResultImpl.java     |  18 +-
 .../gcloud/datastore/StructuredQuery.java     | 448 +++++++++++++-----
 .../gcloud/datastore/SerializationTest.java   |  11 +-
 6 files changed, 462 insertions(+), 206 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index a1ffa9f6ff70..d6827e7d523e 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -116,15 +116,15 @@ static Argument fromPb(DatastoreV1.GqlQueryArg argPb) {
    */
   public static final class Builder {
 
-    private final ResultType resultType;
+    private final ResultClass resultClass;
     private String namespace;
     private String queryString;
     private boolean allowLiteral;
     private Map nameArgs = new TreeMap<>();
     private List numberArgs = new LinkedList<>();
 
-    Builder(ResultType resultType, String query) {
-      this.resultType = resultType;
+    Builder(ResultClass resultClass, String query) {
+      this.resultClass = checkNotNull(resultClass);
       queryString = checkNotNull(query);
     }
 
@@ -268,7 +268,7 @@ private static Argument toArgument(String name, Value.BuilderFactory builderFact
   }
 
   private GqlQuery(Builder builder) {
-    super(builder.resultType, builder.namespace);
+    super(builder.resultClass, builder.namespace);
     queryString = builder.queryString;
     allowLiteral = builder.allowLiteral;
     nameArgs = ImmutableList.copyOf(builder.nameArgs.values());
@@ -367,12 +367,12 @@ protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
   }
 
   @Override
-  protected Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
+  protected Object fromPb(ResultClass resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
     return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
   }
 
-  static  GqlQuery fromPb(ResultType resultType, String namespace,
+  static  GqlQuery fromPb(ResultClass resultType, String namespace,
       DatastoreV1.GqlQuery queryPb) {
     Builder builder = new Builder<>(resultType, queryPb.getQueryString());
     builder.namespace(namespace);
@@ -396,7 +396,7 @@ static  GqlQuery fromPb(ResultType resultType, String namespace,
    * @see GQL Reference
    */
   public static GqlQuery.Builder builder(String gql) {
-    return builder(ResultType.unknown(), gql);
+    return builder(ResultClass.unknown(), gql);
   }
 
   /**
@@ -404,7 +404,7 @@ public static GqlQuery.Builder builder(String gql) {
    *
    * @see GQL Reference
    */
-  public static  GqlQuery.Builder builder(ResultType resultType, String gql) {
-    return new GqlQuery.Builder<>(resultType, gql);
+  public static  GqlQuery.Builder builder(ResultClass resultClass, String gql) {
+    return new GqlQuery.Builder<>(resultClass, gql);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index bf474bc73d1e..4959289e67ec 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -1,11 +1,10 @@
 package com.google.gcloud.datastore;
 
-import static com.google.api.client.util.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
-import com.google.gcloud.datastore.QueryResult.Type;
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 
@@ -21,59 +20,134 @@ public abstract class Query extends Serializable {
 
   private static final long serialVersionUID = -2748141759901313101L;
 
-  private final ResultType resultType;
+  private final ResultClass resultClass;
   private final String namespace;
 
-  public static class ResultType implements java.io.Serializable {
+  public static class ResultClass implements java.io.Serializable {
 
     private static final long serialVersionUID = 2104157695425806623L;
-    private static final ResultType UNKNOWN = new ResultType<>(null, null);
-    private static final ResultType FULL = new ResultType<>(Entity.class, Type.FULL);
-    private static final ResultType KEY_ONLY = new ResultType<>(Key.class, Type.KEY_ONLY);
-    private static final ResultType PROJECTION =
-        new ResultType<>(PartialEntity.class, Type.PROJECTION);
+    private static final ResultClass UNKNOWN = new ResultClass<>(Object.class);
+    private static final ResultClass FULL = new ResultClass<>(Entity.class);
+    private static final ResultClass KEY_ONLY = new ResultClass<>(Key.class);
+    private static final ResultClass PROJECTION =
+        new ResultClass<>(PartialEntity.class);
 
+    private final Class value;
 
-    private final Class clazz;
-    private final Type type;
+    private ResultClass(Class value) {
+      this.value = checkNotNull(value);
+    }
+
+    public Class value() {
+      return value;
+    }
 
-    private ResultType(Class clazz, Type type) {
-      this.clazz = clazz;
-      this.type = type;
+    @Override
+    public int hashCode() {
+      return value.hashCode();
     }
 
-    public Type getType() {
-      return type;
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof ResultClass)) {
+        return false;
+      }
+      ResultClass other = (ResultClass) obj;
+      return value.equals(other.value);
     }
 
-    public Class getResultClass() {
-      return clazz;
+    @Override
+    public String toString() {
+      ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+      toStringHelper.add("value", value);
+      return toStringHelper.toString();
     }
 
-    public static ResultType unknown() {
+    boolean isAssignableFrom(ResultClass resultClass) {
+      return value.isAssignableFrom(resultClass.value);
+    }
+
+    static ResultClass unknown() {
       return UNKNOWN;
     }
 
-    public static ResultType full() {
+    public static ResultClass full() {
       return FULL;
     }
 
-    public static ResultType projection() {
+    public static ResultClass projection() {
       return PROJECTION;
     }
 
-    public static ResultType keyOnly() {
+    public static ResultClass keyOnly() {
       return KEY_ONLY;
     }
   }
 
-  Query(ResultType resultType, String namespace) {
-    this.resultType = checkNotNull(resultType);
+  /**
+   * Possible results types are:
+   *   FULL: A complete {@link Entity}.
+   *   PROJECTION: A partial entity, represented by {@link PartialEntity}.
+   *   KEY_ONLY: An entity's {@link Key}.
+   */
+  public static enum Type {
+
+    FULL {
+      @Override
+      @SuppressWarnings("unchecked")
+      Entity convert(DatastoreV1.Entity value) {
+        return Entity.fromPb(value);
+      }
+
+      @Override
+      ResultClass resultClass() {
+        return ResultClass.full();
+      }
+    },
+
+    PROJECTION  {
+
+      @Override
+      @SuppressWarnings("unchecked")
+      PartialEntity convert(DatastoreV1.Entity value) {
+        return PartialEntity.fromPb(value);
+      }
+
+      @Override
+      ResultClass resultClass() {
+        return ResultClass.projection();
+      }
+    },
+
+    KEY_ONLY {
+
+      @Override
+      @SuppressWarnings("unchecked")
+      Key convert(DatastoreV1.Entity value) {
+        return Key.fromPb(value.getKey());
+      }
+
+      @Override
+      ResultClass resultClass() {
+        return ResultClass.projection();
+      }
+    };
+
+    abstract  T convert(DatastoreV1.Entity value);
+
+    abstract ResultClass resultClass();
+  }
+
+  Query(ResultClass resultClass, String namespace) {
+    this.resultClass = checkNotNull(resultClass);
     this.namespace = namespace;
   }
 
-  public ResultType resultType() {
-    return resultType;
+  ResultClass getResultClass() {
+    return resultClass;
   }
 
   public String namespace() {
@@ -90,10 +164,10 @@ public String toString() {
 
   @Override
   protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    return fromPb(resultType, namespace, bytesPb);
+    return fromPb(resultClass, namespace, bytesPb);
   }
 
-  protected abstract Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
+  protected abstract Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException;
 
   protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb);
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index 9045764077e4..7f46203ff3f0 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -1,55 +1,16 @@
 package com.google.gcloud.datastore;
 
-import com.google.api.services.datastore.DatastoreV1;
-
 import java.util.Iterator;
 
 /**
  * The result of a Google Cloud Datastore query submission.
  * When the result is not typed it is possible to cast it to its appropriate type according to
- * the {@link #getType} value.
+ * the {@link #getType} result.
  *
  * @param V the type of values the result holds.
  */
 public interface QueryResult extends Iterator {
 
-  /**
-   * Possible results types are:
-   *   FULL: A complete {@link Entity}.
-   *   PROJECTION: A partial entity, represented by {@link PartialEntity}.
-   *   KEY_ONLY: An entity's {@link Key}.
-   */
-  enum Type {
-
-    FULL {
-      @Override
-      @SuppressWarnings("unchecked")
-      Entity convert(DatastoreV1.Entity value) {
-        return Entity.fromPb(value);
-      }
-    },
-
-    PROJECTION  {
-
-      @Override
-      @SuppressWarnings("unchecked")
-      PartialEntity convert(DatastoreV1.Entity value) {
-        return PartialEntity.fromPb(value);
-      }
-    },
-
-    KEY_ONLY {
-
-      @Override
-      @SuppressWarnings("unchecked")
-      Key convert(DatastoreV1.Entity value) {
-        return Key.fromPb(value.getKey());
-      }
-    };
-
-    abstract  T convert(DatastoreV1.Entity value);
-  }
-
   /**
    * Returns the actual type of the result's values.
    * When needed the result could be casted accordingly:
@@ -59,7 +20,7 @@ Key convert(DatastoreV1.Entity value) {
    * Type.KEY_ONLY -> (QueryResult)
    * } 
    */
-  Type getType();
+  Query.Type getType();
 
   /**
    * Return the Cursor for the next result. Not currently implemented (depends on v1beta3).
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index 734b8b1bee4a..7e50d0bef5d6 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -9,23 +9,24 @@
 
 class QueryResultImpl implements QueryResult {
 
-  private static final ImmutableMap
+  private static final ImmutableMap
       RESULT_TYPE_CONVERTER;
 
   private final DatastoreServiceImpl datastore;
   private final DatastoreV1.ReadOptions readOptionsPb;
-  private final QueryResult.Type type;
+  private final Query.ResultClass resultClass;
   private Query query;
+  private Query.Type type;
   private DatastoreV1.QueryResultBatch resultPb;
   private Iterator entityResultPbIter;
   private ByteString endCursor;
   private int count;
 
   static {
-    ImmutableMap.Builder builder =
+    ImmutableMap.Builder builder =
         ImmutableMap.builder();
     for (DatastoreV1.EntityResult.ResultType type : DatastoreV1.EntityResult.ResultType.values()) {
-      builder.put(type, QueryResult.Type.valueOf(type.name()));
+      builder.put(type, Query.Type.valueOf(type.name()));
     }
     RESULT_TYPE_CONVERTER = builder.build();
   }
@@ -35,10 +36,8 @@ class QueryResultImpl implements QueryResult {
     this.datastore = datastore;
     this.readOptionsPb = readOptionsPb;
     this.query = query;
+    this.resultClass = query.getResultClass();
     sendRequest();
-    type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
-    Preconditions.checkState(query.resultType().getType() == null
-        || query.resultType().getType() == type, "Unexpected result type");
   }
 
   private DatastoreV1.QueryResultBatch sendRequest() {
@@ -63,6 +62,9 @@ private DatastoreV1.QueryResultBatch sendRequest() {
     } else {
       endCursor = null;
     }
+    type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
+    Preconditions.checkState(resultClass.isAssignableFrom(type.resultClass()),
+        "Unexpected result type");
     return resultPb;
   }
 
@@ -88,7 +90,7 @@ public void remove() {
   }
 
   @Override
-  public QueryResult.Type getType() {
+  public Query.Type getType() {
     return type;
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 684d31a8fcf4..ac5f8519cf5c 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -20,13 +20,14 @@
 import java.util.List;
 import java.util.Objects;
 
-public class StructuredQuery extends Query {
+public abstract class StructuredQuery extends Query {
 
   private static final long serialVersionUID = 546838955624019594L;
 
-  private final transient boolean keysOnly;
   private final transient String kind;
+  private final ImmutableList projection;
   private final transient Filter filter;
+  private final ImmutableList groupBy;
   private final transient ImmutableList orderBy;
   private final transient Cursor startCursor;
   private final transient Cursor endCursor;
@@ -44,8 +45,7 @@ public abstract static class Filter implements Serializable {
     protected abstract DatastoreV1.Filter toPb();
   }
 
-
-  public static final class Expression extends Filter {
+  public static final class CompositeFilter extends Filter {
 
     private static final long serialVersionUID = 3610352685739360009L;
     private final Operator operator;
@@ -59,14 +59,14 @@ DatastoreV1.CompositeFilter.Operator toPb() {
       }
     }
 
-    private Expression(Operator operator, Filter first, Filter... other) {
+    private CompositeFilter(Operator operator, Filter first, Filter... other) {
       this.operator = operator;
       this.filters =
           ImmutableList.builder().add(first).addAll(Arrays.asList(other)).build();
     }
 
-    public static Expression and(Filter first, Filter... other) {
-      return new Expression(Operator.AND, first, other);
+    public static CompositeFilter and(Filter first, Filter... other) {
+      return new CompositeFilter(Operator.AND, first, other);
     }
 
     @Override
@@ -81,7 +81,7 @@ protected DatastoreV1.Filter toPb() {
     }
   }
 
-  public static final class Condition extends Filter {
+  public static final class PropertyFilter extends Filter {
 
     private static final long serialVersionUID = -4514695915258598597L;
 
@@ -108,13 +108,13 @@ public DatastoreV1.PropertyFilter.Operator toPb() {
       }
     }
 
-    private Condition(String property, Operator operator, Value value) {
+    private PropertyFilter(String property, Operator operator, Value value) {
       this.property = checkNotNull(property);
       this.operator = checkNotNull(operator);
       this.value = checkNotNull(value);
     }
 
-    private Condition(String property, Operator operator) {
+    private PropertyFilter(String property, Operator operator) {
       this.property = checkNotNull(property);
       this.operator = checkNotNull(operator);
       this.value = null;
@@ -130,181 +130,181 @@ public boolean equals(Object obj) {
       if (obj == this) {
         return true;
       }
-      if (!(obj instanceof Condition)) {
+      if (!(obj instanceof PropertyFilter)) {
         return false;
       }
-      Condition other = (Condition) obj;
+      PropertyFilter other = (PropertyFilter) obj;
       return property.equals(other.property)
           && operator.equals(other.operator)
           && Objects.equals(value, other.value);
     }
 
-    public static Condition le(String property, Value value) {
-      return new Condition(property, Operator.LESS_THAN, value);
+    public static PropertyFilter le(String property, Value value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, value);
     }
 
-    public static Condition le(String property, String value) {
-      return new Condition(property, Operator.LESS_THAN, of(value));
+    public static PropertyFilter le(String property, String value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Condition le(String property, long value) {
-      return new Condition(property, Operator.LESS_THAN, of(value));
+    public static PropertyFilter le(String property, long value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Condition le(String property, double value) {
-      return new Condition(property, Operator.LESS_THAN, of(value));
+    public static PropertyFilter le(String property, double value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Condition le(String property, boolean value) {
-      return new Condition(property, Operator.LESS_THAN, of(value));
+    public static PropertyFilter le(String property, boolean value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Condition le(String property, DateTime value) {
-      return new Condition(property, Operator.LESS_THAN, of(value));
+    public static PropertyFilter le(String property, DateTime value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Condition le(String property, Key value) {
-      return new Condition(property, Operator.LESS_THAN, of(value));
+    public static PropertyFilter le(String property, Key value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Condition le(String property, Blob value) {
-      return new Condition(property, Operator.LESS_THAN, of(value));
+    public static PropertyFilter le(String property, Blob value) {
+      return new PropertyFilter(property, Operator.LESS_THAN, of(value));
     }
 
-    public static Condition lte(String property, Value value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, value);
+    public static PropertyFilter lte(String property, Value value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, value);
     }
 
-    public static Condition lte(String property, String value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter lte(String property, String value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition lte(String property, long value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter lte(String property, long value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition lte(String property, double value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter lte(String property, double value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition lte(String property, boolean value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter lte(String property, boolean value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition lte(String property, DateTime value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter lte(String property, DateTime value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition lte(String property, Key value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter lte(String property, Key value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition lte(String property, Blob value) {
-      return new Condition(property, Operator.LESS_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter lte(String property, Blob value) {
+      return new PropertyFilter(property, Operator.LESS_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition gt(String property, Value value) {
-      return new Condition(property, Operator.GREATER_THAN, value);
+    public static PropertyFilter gt(String property, Value value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, value);
     }
 
-    public static Condition gt(String property, String value) {
-      return new Condition(property, Operator.GREATER_THAN, of(value));
+    public static PropertyFilter gt(String property, String value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Condition gt(String property, long value) {
-      return new Condition(property, Operator.GREATER_THAN, of(value));
+    public static PropertyFilter gt(String property, long value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Condition gt(String property, double value) {
-      return new Condition(property, Operator.GREATER_THAN, of(value));
+    public static PropertyFilter gt(String property, double value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Condition gt(String property, boolean value) {
-      return new Condition(property, Operator.GREATER_THAN, of(value));
+    public static PropertyFilter gt(String property, boolean value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Condition gt(String property, DateTime value) {
-      return new Condition(property, Operator.GREATER_THAN, of(value));
+    public static PropertyFilter gt(String property, DateTime value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Condition gt(String property, Key value) {
-      return new Condition(property, Operator.GREATER_THAN, of(value));
+    public static PropertyFilter gt(String property, Key value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Condition gt(String property, Blob value) {
-      return new Condition(property, Operator.GREATER_THAN, of(value));
+    public static PropertyFilter gt(String property, Blob value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN, of(value));
     }
 
-    public static Condition gte(String property, Value value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, value);
+    public static PropertyFilter gte(String property, Value value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, value);
     }
 
-    public static Condition gte(String property, String value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter gte(String property, String value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition gte(String property, long value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter gte(String property, long value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition gte(String property, double value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter gte(String property, double value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition gte(String property, boolean value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter gte(String property, boolean value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition gte(String property, DateTime value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter gte(String property, DateTime value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition gte(String property, Key value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter gte(String property, Key value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition gte(String property, Blob value) {
-      return new Condition(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
+    public static PropertyFilter gte(String property, Blob value) {
+      return new PropertyFilter(property, Operator.GREATER_THAN_OR_EQUAL, of(value));
     }
 
-    public static Condition eq(String property, Value value) {
-      return new Condition(property, Operator.EQUAL, value);
+    public static PropertyFilter eq(String property, Value value) {
+      return new PropertyFilter(property, Operator.EQUAL, value);
     }
 
-    public static Condition eq(String property, String value) {
-      return new Condition(property, Operator.EQUAL, of(value));
+    public static PropertyFilter eq(String property, String value) {
+      return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static Condition eq(String property, long value) {
-      return new Condition(property, Operator.EQUAL, of(value));
+    public static PropertyFilter eq(String property, long value) {
+      return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static Condition eq(String property, double value) {
-      return new Condition(property, Operator.EQUAL, of(value));
+    public static PropertyFilter eq(String property, double value) {
+      return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static Condition eq(String property, boolean value) {
-      return new Condition(property, Operator.EQUAL, of(value));
+    public static PropertyFilter eq(String property, boolean value) {
+      return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static Condition eq(String property, DateTime value) {
-      return new Condition(property, Operator.EQUAL, of(value));
+    public static PropertyFilter eq(String property, DateTime value) {
+      return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static Condition eq(String property, Key value) {
-      return new Condition(property, Operator.EQUAL, of(value));
+    public static PropertyFilter eq(String property, Key value) {
+      return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static Condition eq(String property, Blob value) {
-      return new Condition(property, Operator.EQUAL, of(value));
+    public static PropertyFilter eq(String property, Blob value) {
+      return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static Condition hasAncestor(String property) {
-      return new Condition(property, Operator.HAS_ANCESTOR);
+    public static PropertyFilter hasAncestor(String property) {
+      return new PropertyFilter(property, Operator.HAS_ANCESTOR);
     }
 
-    public static Condition isNull(String property) {
-      return new Condition(property, Operator.IS_NULL, NullValue.of());
+    public static PropertyFilter isNull(String property) {
+      return new PropertyFilter(property, Operator.IS_NULL, NullValue.of());
     }
 
     @Override
@@ -371,16 +371,23 @@ public static OrderBy desc(String property) {
     }
   }
 
-  static class Builder> {
+  abstract static class BaseBuilder> {
 
+    private ResultClass resultClass;
     private String namespace;
     private String kind;
+    private List projection = new LinkedList<>();
+    private Filter filter;
+    private List groupBy = new LinkedList<>();
+    private List orderBy = new LinkedList<>();
     private Cursor startCursor;
     private Cursor endCursor;
     private Integer offset;
     private Integer limit;
-    private Filter filter;
-    private List orderBy = new LinkedList<>();
+
+    BaseBuilder(ResultClass resultClass) {
+      this.resultClass = resultClass;
+    }
 
     @SuppressWarnings("unchecked")
     protected B self() {
@@ -429,6 +436,12 @@ public B clearOrderBy() {
       return self();
     }
 
+    public B orderBy(OrderBy orderBy, OrderBy... others) {
+      clearOrderBy();
+      addOrderBy(orderBy, others);
+      return self();
+    }
+
     public B addOrderBy(OrderBy orderBy, OrderBy... others) {
       this.orderBy.add(orderBy);
       for (OrderBy other : others) {
@@ -436,26 +449,172 @@ public B addOrderBy(OrderBy orderBy, OrderBy... others) {
       }
       return self();
     }
+
+    protected B clearProjection() {
+      projection.clear();
+      return self();
+    }
+
+    protected B projection(String property, String... others) {
+      clearProjection();
+      addProjection(property, others);
+      return self();
+    }
+
+    protected B addProjection(String property, String... others) {
+      this.projection.add(property);
+      for (String other : others) {
+        this.projection.add(other);
+      }
+      return self();
+    }
+
+    protected B clearGroupBy() {
+      groupBy.clear();
+      return self();
+    }
+
+    protected B groupBy(String property, String... others) {
+      clearGroupBy();
+      addGroupBy(property, others);
+      return self();
+    }
+
+    protected B addGroupBy(String property, String... others) {
+      this.groupBy.add(property);
+      for (String other : others) {
+        this.groupBy.add(other);
+      }
+      return self();
+    }
+
+    public abstract StructuredQuery build();
+  }
+
+  public static final class FullQueryBuilder extends BaseBuilder {
+
+    FullQueryBuilder() {
+      super(ResultClass.full());
+    }
+
+    @Override
+    public FullQuery build() {
+      return new FullQuery(this);
+    }
+  }
+
+  public static final class KeyOnlyQueryBuilder extends BaseBuilder {
+
+    public KeyOnlyQueryBuilder() {
+      super(ResultClass.keyOnly());
+      projection("__key__");
+    }
+
+    @Override
+    public KeyOnlyQuery build() {
+      return new KeyOnlyQuery(this);
+    }
+  }
+
+  public static final class ProjectionQueryBuilder
+      extends BaseBuilder {
+
+    public ProjectionQueryBuilder() {
+      super(ResultClass.projection());
+    }
+
+    @Override
+    public ProjectionQuery build() {
+      return new ProjectionQuery(this);
+    }
+
+    @Override
+    public ProjectionQueryBuilder clearProjection() {
+      return super.clearProjection();
+    }
+
+    @Override
+    public ProjectionQueryBuilder projection(String property, String... others) {
+      return super.projection(property, others);
+    }
+
+    @Override
+    public ProjectionQueryBuilder addProjection(String property, String... others) {
+      return super.addProjection(property, others);
+    }
+
+    @Override
+    public ProjectionQueryBuilder clearGroupBy() {
+      return super.clearGroupBy();
+    }
+
+    @Override
+    public ProjectionQueryBuilder groupBy(String property, String... others) {
+      return super.groupBy(property, others);
+    }
+
+    @Override
+    public ProjectionQueryBuilder addGroupBy(String property, String... others) {
+      return super.addGroupBy(property, others);
+    }
+  }
+
+  public static final class FullQuery extends StructuredQuery {
+
+    private static final long serialVersionUID = -84461800292593840L;
+
+    FullQuery(FullQueryBuilder builder) {
+      super(builder);
+    }
+  }
+
+  public static final class KeyOnlyQuery extends StructuredQuery {
+
+    private static final long serialVersionUID = -7502917784216095473L;
+
+    KeyOnlyQuery(KeyOnlyQueryBuilder builder) {
+      super(builder);
+    }
   }
 
-  private StructuredQuery(ResultType resultType, boolean keysOnly, String namespace, String kind,
-      Cursor startCursor, Cursor endCursor, Integer offset, Integer limit,
-      Filter filter, ImmutableList orderBy) {
-    super(resultType, namespace);
-    this.keysOnly = keysOnly;
-    this.kind = kind;
-    this.startCursor = startCursor;
-    this.endCursor = endCursor;
-    this.offset = offset;
-    this.limit = limit;
-    this.filter = filter;
-    this.orderBy = orderBy;
+  public static final class ProjectionQuery extends StructuredQuery {
+
+    private static final long serialVersionUID = -3333183044486150649L;
+
+    ProjectionQuery(ProjectionQueryBuilder builder) {
+      super(builder);
+      Preconditions.checkState(!keyOnly(),
+          "Projection query can't project only '__key__', use KeyOnlyQuery instead.");
+    }
+
+    @Override
+    public List projection() {
+      return super.projection();
+    }
+
+    @Override
+    public List groupBy() {
+      return super.groupBy();
+    }
+  }
+
+  StructuredQuery(BaseBuilder builder) {
+    super(builder.resultClass, builder.namespace);
+    kind = builder.kind;
+    projection = ImmutableList.copyOf(builder.projection);
+    filter = builder.filter;
+    groupBy = ImmutableList.copyOf(builder.groupBy);
+    orderBy = ImmutableList.copyOf(builder.orderBy);
+    startCursor = builder.startCursor;
+    endCursor = builder.endCursor;
+    offset = builder.offset;
+    limit = builder.limit;
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(
-        namespace(), keysOnly, kind, startCursor, endCursor, offset, limit, filter, orderBy);
+    return Objects.hash(namespace(), kind, startCursor, endCursor, offset, limit, filter, orderBy,
+        projection(), groupBy());
   }
 
   @Override
@@ -474,7 +633,50 @@ public boolean equals(Object obj) {
         && Objects.equals(offset, other.offset)
         && Objects.equals(limit, other.limit)
         && Objects.equals(filter, other.filter)
-        && Objects.equals(orderBy, other.orderBy);
+        && Objects.equals(orderBy, other.orderBy)
+        && Objects.equals(projection, other.projection)
+        && Objects.equals(groupBy, other.groupBy);
+
+  }
+
+  public String kind() {
+    return kind;
+  }
+
+  protected boolean keyOnly() {
+    return projection.size() == 1 && projection.get(0).equals("__key__");
+  }
+
+  protected List projection() {
+    return projection;
+  }
+
+  public Filter filter() {
+    return filter;
+  }
+
+  protected List groupBy() {
+    return groupBy;
+  }
+
+  public ImmutableList orderBy() {
+    return orderBy;
+  }
+
+  public Cursor etartCursor() {
+    return startCursor;
+  }
+
+  public Cursor endCursor() {
+    return endCursor;
+  }
+
+  public Integer offset() {
+    return offset;
+  }
+
+  public Integer limit() {
+    return limit;
   }
 
   @Override
@@ -489,9 +691,9 @@ protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
   }
 
   @Override
-  protected Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
+  protected Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
-    return fromPb(resultType, namespace, DatastoreV1.Query.parseFrom(bytesPb));
+    return fromPb(resultClass, namespace, DatastoreV1.Query.parseFrom(bytesPb));
   }
 
   @Override
@@ -519,9 +721,21 @@ protected DatastoreV1.Query toPb() {
     return queryPb.build();
   }
 
-  static  StructuredQuery fromPb(ResultType resultType, String namespace,
+  static  StructuredQuery fromPb(ResultClass resultType, String namespace,
       DatastoreV1.Query queryPb) {
     // TODO: implement
     return null;
   }
+
+  public static FullQueryBuilder builder() {
+    return new FullQueryBuilder();
+  }
+
+  public static KeyOnlyQueryBuilder keyOnlyBuilder() {
+    return new KeyOnlyQueryBuilder();
+  }
+
+  public static ProjectionQueryBuilder projectionBuilder() {
+    return new ProjectionQueryBuilder();
+  }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index ce3ea54f0a85..4c9f1fdad3c8 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -7,7 +7,7 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Multimap;
-import com.google.gcloud.datastore.Query.ResultType;
+import com.google.gcloud.datastore.Query.ResultClass;
 import com.google.gcloud.datastore.Value.Type;
 
 import org.junit.Test;
@@ -35,11 +35,16 @@ public class SerializationTest {
       .namespace("ns1")
       .build();
   private static final Query GQL2 =
-      GqlQuery.builder(ResultType.full(), "select * from kind1 where name = @name and age > @1")
+      GqlQuery.builder(ResultClass.full(), "select * from kind1 where name = @name and age > @1")
       .setArgument("name", "name1")
       .addArgument(20)
       .namespace("ns1")
       .build();
+  private static final Query QUERY1 = StructuredQuery.builder().kind("kind1").build();
+  private static final Query QUERY2 = StructuredQuery.keyOnlyBuilder().kind("k").filter(
+      StructuredQuery.PropertyFilter.eq("p1", "hello")).build();
+  private static final Query QUERY3 =
+      StructuredQuery.projectionBuilder().kind("k").projection("p").build();
   private static final KeyValue KEY_VALUE = KeyValue.of(KEY1);
   private static final NullValue NULL_VALUE = NullValue.builder().indexed(true).build();
   private static final StringValue STRING_VALUE = StringValue.of("hello");
@@ -108,7 +113,7 @@ public void testValues() throws Exception {
   public void testTypes() throws Exception {
     Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
         ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3, DATE_TIME1,
-        BLOB1, CURSOR1, GQL1, GQL2};
+        BLOB1, CURSOR1, GQL1, GQL2, QUERY1, QUERY2, QUERY3};
     for (Object obj : types) {
       Object copy = serialiazeAndDeserialize(obj);
       assertEquals(obj, obj);

From 805d2ea724d4847bfe3f6726d4746573973a3b8e Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 14 Dec 2014 23:07:54 -0800
Subject: [PATCH 055/732] structured query work in progress

---
 .../google/gcloud/datastore/QueryResult.java  |  20 +-
 .../gcloud/datastore/QueryResultImpl.java     |  38 +--
 .../gcloud/datastore/StructuredQuery.java     | 272 +++++++++++++++---
 .../gcloud/datastore/SerializationTest.java   |   3 +-
 4 files changed, 248 insertions(+), 85 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index 7f46203ff3f0..c569fc7bd361 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -4,26 +4,20 @@
 
 /**
  * The result of a Google Cloud Datastore query submission.
- * When the result is not typed it is possible to cast it to its appropriate type according to
- * the {@link #getType} result.
+ * When result is not typed it is possible to cast it to its appropriate type according to
+ * the {@link #resultClass} value.
  *
- * @param V the type of values the result holds.
+ * @param V the type of the results value.
  */
 public interface QueryResult extends Iterator {
 
   /**
-   * Returns the actual type of the result's values.
-   * When needed the result could be casted accordingly:
-   * 
 {@code
-   * Type.FULL -> (QueryResult)
-   * Type.PROJECTION -> (QueryResult)
-   * Type.KEY_ONLY -> (QueryResult)
-   * } 

+   * Returns the actual class of the result's values.
    */
-  Query.Type getType();
+  Class resultClass();
 
   /**
-   * Return the Cursor for the next result. Not currently implemented (depends on v1beta3).
+   * Returns the Cursor for the next result. Not currently implemented (depends on v1beta3).
    */
-  Cursor getCursor();
+  Cursor cursor();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index 7e50d0bef5d6..3f81a7fd0010 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -2,12 +2,12 @@
 
 import com.google.api.client.repackaged.com.google.common.base.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.ImmutableMap;
-import com.google.protobuf.ByteString;
 
 import java.util.Iterator;
 
-class QueryResultImpl implements QueryResult {
+class QueryResultImpl extends AbstractIterator implements QueryResult {
 
   private static final ImmutableMap
       RESULT_TYPE_CONVERTER;
@@ -19,8 +19,6 @@ class QueryResultImpl implements QueryResult {
   private Query.Type type;
   private DatastoreV1.QueryResultBatch resultPb;
   private Iterator entityResultPbIter;
-  private ByteString endCursor;
-  private int count;
 
   static {
     ImmutableMap.Builder builder =
@@ -57,11 +55,6 @@ private DatastoreV1.QueryResultBatch sendRequest() {
     query.populatePb(requestPb);
     resultPb = datastore.runQuery(requestPb.build()).getBatch();
     entityResultPbIter = resultPb.getEntityResultList().iterator();
-    if (DatastoreV1.QueryResultBatch.MoreResultsType.NOT_FINISHED == resultPb.getMoreResults()) {
-      endCursor = resultPb.getEndCursor();
-    } else {
-      endCursor = null;
-    }
     type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
     Preconditions.checkState(resultClass.isAssignableFrom(type.resultClass()),
         "Unexpected result type");
@@ -69,33 +62,24 @@ private DatastoreV1.QueryResultBatch sendRequest() {
   }
 
   @Override
-  public boolean hasNext() {
-    return entityResultPbIter.hasNext() || endCursor != null;
-  }
-
-  @Override
-  public T next() {
-    if (!hasNext() && endCursor != null) {
+  protected T computeNext() {
+    while (!entityResultPbIter.hasNext()
+        && resultPb.getMoreResults() == DatastoreV1.QueryResultBatch.MoreResultsType.NOT_FINISHED) {
       query = query.nextQuery(resultPb);
       sendRequest();
     }
-    DatastoreV1.Entity entity = entityResultPbIter.next().getEntity();
-    count++;
-    return type.convert(entity);
-  }
-
-  @Override
-  public void remove() {
-    throw new UnsupportedOperationException("QueryResult is read-only");
+    return entityResultPbIter.hasNext()
+        ? type.convert(entityResultPbIter.next().getEntity())
+        : endOfData();
   }
 
   @Override
-  public Query.Type getType() {
-    return type;
+  public Class resultClass() {
+    return type.resultClass().value();
   }
 
   @Override
-  public Cursor getCursor() {
+  public Cursor cursor() {
     // TODO(ozarov): implement when v1beta3 is available
     return null;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index ac5f8519cf5c..a4d92290e964 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -9,8 +9,10 @@
 import static com.google.gcloud.datastore.LongValue.of;
 import static com.google.gcloud.datastore.StringValue.of;
 
-import com.google.api.client.util.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.InvalidProtocolBufferException;
 
@@ -23,9 +25,10 @@
 public abstract class StructuredQuery extends Query {
 
   private static final long serialVersionUID = 546838955624019594L;
+  private static final String KEY_PROPERTY_NAME = "__key__";
 
   private final transient String kind;
-  private final ImmutableList projection;
+  private final ImmutableList projection;
   private final transient Filter filter;
   private final ImmutableList groupBy;
   private final transient ImmutableList orderBy;
@@ -34,7 +37,6 @@ public abstract class StructuredQuery extends Query {
   private final transient Integer offset;
   private final transient Integer limit;
 
-
   public abstract static class Filter implements Serializable {
 
     private static final long serialVersionUID = -6443285436239990860L;
@@ -43,6 +45,13 @@ public abstract static class Filter implements Serializable {
     }
 
     protected abstract DatastoreV1.Filter toPb();
+
+    static Filter fromPb(DatastoreV1.Filter filterPb) {
+      if (filterPb.hasCompositeFilter()) {
+        return CompositeFilter.fromPb(filterPb.getCompositeFilter());
+      }
+      return PropertyFilter.fromPb(filterPb.getPropertyFilter());
+    }
   }
 
   public static final class CompositeFilter extends Filter {
@@ -57,6 +66,10 @@ enum Operator {
       DatastoreV1.CompositeFilter.Operator toPb() {
         return DatastoreV1.CompositeFilter.Operator.valueOf(name());
       }
+
+      static Operator fromPb(DatastoreV1.CompositeFilter.Operator operatorPb) {
+        return valueOf(operatorPb.name());
+      }
     }
 
     private CompositeFilter(Operator operator, Filter first, Filter... other) {
@@ -65,6 +78,21 @@ private CompositeFilter(Operator operator, Filter first, Filter... other) {
           ImmutableList.builder().add(first).addAll(Arrays.asList(other)).build();
     }
 
+    private CompositeFilter(Operator operator, ImmutableList filters) {
+      this.operator = operator;
+      this.filters = filters;
+      Preconditions.checkArgument(!filters.isEmpty(), "filters list must not be empty");
+    }
+
+    static CompositeFilter fromPb(DatastoreV1.CompositeFilter compositeFilterPb) {
+      Operator operator = Operator.fromPb(compositeFilterPb.getOperator());
+      ImmutableList.Builder filters = ImmutableList.builder();
+      for (DatastoreV1.Filter filterPb : compositeFilterPb.getFilterList()) {
+        filters.add(Filter.fromPb(filterPb));
+      }
+      return new CompositeFilter(operator, filters.build());
+    }
+
     public static CompositeFilter and(Filter first, Filter... other) {
       return new CompositeFilter(Operator.AND, first, other);
     }
@@ -95,17 +123,15 @@ enum Operator {
       GREATER_THAN,
       GREATER_THAN_OR_EQUAL,
       EQUAL,
-      HAS_ANCESTOR,
-      IS_NULL {
-        @Override
-        public DatastoreV1.PropertyFilter.Operator toPb() {
-          return DatastoreV1.PropertyFilter.Operator.valueOf(EQUAL.name());
-        }
-      };
-
-      public DatastoreV1.PropertyFilter.Operator toPb() {
+      HAS_ANCESTOR;
+
+      DatastoreV1.PropertyFilter.Operator toPb() {
         return DatastoreV1.PropertyFilter.Operator.valueOf(name());
       }
+
+      static Operator fromPb(DatastoreV1.PropertyFilter.Operator operatorPb) {
+        return valueOf(operatorPb.name());
+      }
     }
 
     private PropertyFilter(String property, Operator operator, Value value) {
@@ -114,10 +140,11 @@ private PropertyFilter(String property, Operator operator, Value value) {
       this.value = checkNotNull(value);
     }
 
-    private PropertyFilter(String property, Operator operator) {
-      this.property = checkNotNull(property);
-      this.operator = checkNotNull(operator);
-      this.value = null;
+    public static PropertyFilter fromPb(DatastoreV1.PropertyFilter propertyFilterPb) {
+      String property = propertyFilterPb.getProperty().getName();
+      Operator operator = Operator.fromPb(propertyFilterPb.getOperator());
+      Value value = Value.fromPb(propertyFilterPb.getValue());
+      return new PropertyFilter(property, operator, value);
     }
 
     @Override
@@ -299,12 +326,12 @@ public static PropertyFilter eq(String property, Blob value) {
       return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static PropertyFilter hasAncestor(String property) {
-      return new PropertyFilter(property, Operator.HAS_ANCESTOR);
+    public static PropertyFilter hasAncestor(String property, Key key) {
+      return new PropertyFilter(property, Operator.HAS_ANCESTOR, of(key));
     }
 
     public static PropertyFilter isNull(String property) {
-      return new PropertyFilter(property, Operator.IS_NULL, NullValue.of());
+      return new PropertyFilter(property, Operator.EQUAL, NullValue.of());
     }
 
     @Override
@@ -328,7 +355,16 @@ public static final class OrderBy implements Serializable {
     private final Direction direction;
 
     public enum Direction {
-      ASC, DESC
+
+      ASCENDING, DESCENDING;
+
+      DatastoreV1.PropertyOrder.Direction toPb() {
+        return DatastoreV1.PropertyOrder.Direction.valueOf(name());
+      }
+
+      static Direction fromPb(DatastoreV1.PropertyOrder.Direction directionPb) {
+        return valueOf(directionPb.name());
+      }
     }
 
     public OrderBy(String property, Direction direction) {
@@ -362,21 +398,120 @@ public Direction direction() {
       return direction;
     }
 
+    DatastoreV1.PropertyOrder toPb() {
+      return DatastoreV1.PropertyOrder.newBuilder()
+          .setDirection(direction.toPb())
+          .setProperty(DatastoreV1.PropertyReference.newBuilder().setName(property).build())
+          .build();
+    }
+
     public static OrderBy asc(String property) {
-      return new OrderBy(property, OrderBy.Direction.ASC);
+      return new OrderBy(property, OrderBy.Direction.ASCENDING);
     }
 
     public static OrderBy desc(String property) {
-      return new OrderBy(property, OrderBy.Direction.DESC);
+      return new OrderBy(property, OrderBy.Direction.DESCENDING);
+    }
+
+    static OrderBy fromPb(DatastoreV1.PropertyOrder propertyOrderPb) {
+      String property = propertyOrderPb.getProperty().getName();
+      Direction direction = Direction.fromPb(propertyOrderPb.getDirection());
+      return new OrderBy(property, direction);
+    }
+  }
+
+  public static final class Projection implements Serializable {
+
+    private static final long serialVersionUID = 3083707957256279470L;
+
+    private final Aggregate aggregate;
+    private final String property;
+
+    public enum Aggregate {
+
+      FIRST;
+
+      DatastoreV1.PropertyExpression.AggregationFunction toPb() {
+        return DatastoreV1.PropertyExpression.AggregationFunction.valueOf(name());
+      }
+
+      static Aggregate fromPb(DatastoreV1.PropertyExpression.AggregationFunction aggregatePb) {
+        return valueOf(aggregatePb.name());
+      }
+    }
+
+    private Projection(Aggregate aggregate, String property) {
+      this.aggregate = aggregate;
+      this.property = property;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(property, aggregate);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof Projection)) {
+        return false;
+      }
+      Projection other = (Projection) obj;
+      return Objects.equals(property, other.property)
+          && Objects.equals(aggregate, other.aggregate);
+    }
+
+    @Override
+    public String toString() {
+      ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+      toStringHelper.add("property", property);
+      if (aggregate != null) {
+        toStringHelper.add("aggregate", aggregate);
+      }
+      return toStringHelper.toString();
+    }
+
+    DatastoreV1.PropertyExpression toPb() {
+      DatastoreV1.PropertyExpression.Builder expressionPb =
+          DatastoreV1.PropertyExpression.newBuilder();
+      if (aggregate != null) {
+        expressionPb.setAggregationFunction(aggregate.toPb());
+      }
+      expressionPb.setProperty(
+          DatastoreV1.PropertyReference.newBuilder().setName(property).build());
+      return expressionPb.build();
+    }
+
+    public static Projection fromPb(DatastoreV1.PropertyExpression propertyExpressionPb) {
+      String property = propertyExpressionPb.getProperty().getName();
+      Aggregate aggregate = null;
+      if (propertyExpressionPb.hasAggregationFunction()) {
+        aggregate = Aggregate.fromPb(propertyExpressionPb.getAggregationFunction());
+      }
+      return new Projection(aggregate, property);
+    }
+
+    public static Projection property(String property) {
+      return new Projection(null, property);
+    }
+
+    public static Projection aggregate(Aggregate aggregate, String property) {
+      return new Projection(aggregate, property);
+    }
+
+    public static Projection first(String property) {
+      return new Projection(Aggregate.FIRST, property);
     }
   }
 
-  abstract static class BaseBuilder> {
+  static abstract class BaseBuilder> {
 
     private ResultClass resultClass;
     private String namespace;
     private String kind;
-    private List projection = new LinkedList<>();
+    private List projection = new LinkedList<>();
     private Filter filter;
     private List groupBy = new LinkedList<>();
     private List orderBy = new LinkedList<>();
@@ -409,7 +544,7 @@ public B startCursor(Cursor startCursor) {
       return self();
     }
 
-    public B encCursor(Cursor endCursor) {
+    public B endCursor(Cursor endCursor) {
       this.endCursor = endCursor;
       return self();
     }
@@ -455,15 +590,15 @@ protected B clearProjection() {
       return self();
     }
 
-    protected B projection(String property, String... others) {
+    protected B projection(Projection projection, Projection... others) {
       clearProjection();
-      addProjection(property, others);
+      addProjection(projection, others);
       return self();
     }
 
-    protected B addProjection(String property, String... others) {
-      this.projection.add(property);
-      for (String other : others) {
+    protected B addProjection(Projection projection, Projection... others) {
+      this.projection.add(projection);
+      for (Projection other : others) {
         this.projection.add(other);
       }
       return self();
@@ -488,6 +623,37 @@ protected B addGroupBy(String property, String... others) {
       return self();
     }
 
+    protected B mergeFrom(DatastoreV1.Query queryPb) {
+      if (queryPb.getKindCount() > 0) {
+        kind(queryPb.getKind(0).getName());
+      }
+      if (queryPb.hasStartCursor()) {
+        startCursor(new Cursor(queryPb.getStartCursor()));
+      }
+      if (queryPb.hasEndCursor()) {
+        endCursor(new Cursor(queryPb.getEndCursor()));
+      }
+      if (queryPb.hasOffset()) {
+        offset(queryPb.getOffset());
+      }
+      if (queryPb.hasLimit()) {
+        limit(queryPb.getLimit());
+      }
+      if (queryPb.hasFilter()) {
+        filter(Filter.fromPb(queryPb.getFilter()));
+      }
+      for (DatastoreV1.PropertyOrder orderByPb : queryPb.getOrderList()) {
+        addOrderBy(OrderBy.fromPb(orderByPb));
+      }
+      for (DatastoreV1.PropertyExpression projectionPb : queryPb.getProjectionList()) {
+        addProjection(Projection.fromPb(projectionPb));
+      }
+      for (DatastoreV1.PropertyReference groupByPb : queryPb.getGroupByList()) {
+        addGroupBy(groupByPb.getName());
+      }
+      return self();
+    }
+
     public abstract StructuredQuery build();
   }
 
@@ -499,6 +665,8 @@ public static final class FullQueryBuilder extends BaseBuilder
     }
 
     @Override
-    public List projection() {
+    public List projection() {
       return super.projection();
     }
 
@@ -644,10 +813,10 @@ public String kind() {
   }
 
   protected boolean keyOnly() {
-    return projection.size() == 1 && projection.get(0).equals("__key__");
+    return projection.size() == 1 && projection.get(0).property.equals(KEY_PROPERTY_NAME);
   }
 
-  protected List projection() {
+  protected List projection() {
     return projection;
   }
 
@@ -663,7 +832,7 @@ public ImmutableList orderBy() {
     return orderBy;
   }
 
-  public Cursor etartCursor() {
+  public Cursor startCursor() {
     return startCursor;
   }
 
@@ -685,7 +854,7 @@ protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
   }
 
   @Override
-  protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
+  protected StructuredQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
     // TODO: implment
     throw new UnsupportedOperationException("paging not implemented yet");
   }
@@ -717,14 +886,29 @@ protected DatastoreV1.Query toPb() {
     if (filter != null) {
       queryPb.setFilter(filter.toPb());
     }
-    // TODO: projection, groupBy, orderBy (or keys-only)
+    for (OrderBy value : orderBy) {
+      queryPb.addOrder(value.toPb());
+    }
+    for (String value : groupBy) {
+      queryPb.addGroupBy(DatastoreV1.PropertyReference.newBuilder().setName(value).build());
+    }
+    for (Projection value : projection) {
+      queryPb.addProjection(value.toPb());
+    }
     return queryPb.build();
   }
 
-  static  StructuredQuery fromPb(ResultClass resultType, String namespace,
+  static StructuredQuery fromPb(ResultClass resultClass, String namespace,
       DatastoreV1.Query queryPb) {
-    // TODO: implement
-    return null;
+    BaseBuilder builder;
+    if (resultClass.equals(ResultClass.full())) {
+      builder = new FullQueryBuilder();
+    } else if (resultClass.equals(ResultClass.keyOnly())) {
+      builder = new KeyOnlyQueryBuilder();
+    } else {
+      builder = new ProjectionQueryBuilder();
+    }
+    return builder.namespace(namespace).mergeFrom(queryPb).build();
   }
 
   public static FullQueryBuilder builder() {
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 4c9f1fdad3c8..79c08cffa674 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -8,6 +8,7 @@
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Multimap;
 import com.google.gcloud.datastore.Query.ResultClass;
+import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.Value.Type;
 
 import org.junit.Test;
@@ -44,7 +45,7 @@ public class SerializationTest {
   private static final Query QUERY2 = StructuredQuery.keyOnlyBuilder().kind("k").filter(
       StructuredQuery.PropertyFilter.eq("p1", "hello")).build();
   private static final Query QUERY3 =
-      StructuredQuery.projectionBuilder().kind("k").projection("p").build();
+      StructuredQuery.projectionBuilder().kind("k").projection(Projection.property("p")).build();
   private static final KeyValue KEY_VALUE = KeyValue.of(KEY1);
   private static final NullValue NULL_VALUE = NullValue.builder().indexed(true).build();
   private static final StringValue STRING_VALUE = StringValue.of("hello");

From 0e4ec0896bdf7893cc946e9a8209fa2f69f6eb9d Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 15 Dec 2014 18:06:17 -0800
Subject: [PATCH 056/732] complete query and start adding tests/javadoc

---
 .classpath                                    |  10 ++
 .../com/google/gcloud/datastore/GqlQuery.java | 109 ++++++------
 .../gcloud/datastore/PartialEntity.java       |   6 +-
 .../com/google/gcloud/datastore/Query.java    |  30 ++--
 .../google/gcloud/datastore/QueryResult.java  |   2 +-
 .../gcloud/datastore/QueryResultImpl.java     |  34 ++--
 .../gcloud/datastore/StructuredQuery.java     | 156 +++++++++++-------
 .../datastore/DatastoreServiceTest.java       |  69 +++++++-
 .../gcloud/datastore/SerializationTest.java   |  24 ++-
 9 files changed, 292 insertions(+), 148 deletions(-)

diff --git a/.classpath b/.classpath
index 9ed6ee4d0713..d6cf6121af66 100644
--- a/.classpath
+++ b/.classpath
@@ -23,5 +23,15 @@
 		
 	
 	
+	
+		
+			
+		
+	
+	
+		
+			
+		
+	
 	
 
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index d6827e7d523e..ff8633760f87 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -24,9 +24,33 @@
 /**
  * A Google Cloud Datastore GQL.
  *
+ * 
A usage example:

+ *
+ * When the type of the results is known the preferred usage would be:
+ * 
 {@code
+ *   Query query = GqlQuery.builder(ResultClass.full(), "select * from kind").build();
+ *   QueryResult results = datastore.runQuery(query);
+ *   while (results.hasNext()) {
+ *     Entity entity = results.next();
+ *     ...
+ *   }
+ * } 

+ *
+ * When the type of the results is unknown you can use this approach:
+ * 
 {@code
+ *   Query query = GqlQuery.builder("select __key__ from kind").build();
+ *   QueryResult results = datastore.runQuery(query);
+ *   if (Key.class.isAssignableFrom(results.resultClass())) {
+ *     QueryResult keys = (QueryResult) results;
+ *     while (keys.hasNext()) {
+ *       Key key = keys.next();
+ *       ...
+ *     }
+ *   }
+ * } 

  * @see GQL Reference
  */
-public final class GqlQuery extends Query {
+public final class GqlQuery extends Query {
 
   private static final long serialVersionUID = 5988280590929540569L;
 
@@ -114,132 +138,132 @@ static Argument fromPb(DatastoreV1.GqlQueryArg argPb) {
   /**
    * A GQL query builder.
    */
-  public static final class Builder {
+  public static final class Builder {
 
-    private final ResultClass resultClass;
+    private final ResultClass resultClass;
     private String namespace;
     private String queryString;
     private boolean allowLiteral;
     private Map nameArgs = new TreeMap<>();
     private List numberArgs = new LinkedList<>();
 
-    Builder(ResultClass resultClass, String query) {
+    Builder(ResultClass resultClass, String query) {
       this.resultClass = checkNotNull(resultClass);
       queryString = checkNotNull(query);
     }
 
-    public Builder query(String query) {
+    public Builder query(String query) {
       queryString = checkNotNull(query);
       return this;
     }
 
-    public Builder namespace(String namespace) {
+    public Builder namespace(String namespace) {
       this.namespace = validateNamespace(namespace);
       return this;
     }
 
-    public Builder allowLiteral(boolean allowLiteral) {
+    public Builder allowLiteral(boolean allowLiteral) {
       this.allowLiteral = allowLiteral;
       return this;
     }
 
-    public Builder clearArguments() {
+    public Builder clearArguments() {
       nameArgs.clear();
       numberArgs.clear();
       return this;
     }
 
-    public Builder setArgument(String name, Cursor cursor) {
+    public Builder setArgument(String name, Cursor cursor) {
       nameArgs.put(name, new Argument(name, cursor));
       return this;
     }
 
-    public Builder setArgument(String name, String... value) {
+    public Builder setArgument(String name, String... value) {
       nameArgs.put(name, toArgument(name, StringValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, long... value) {
+    public Builder setArgument(String name, long... value) {
       nameArgs.put(name, toArgument(name, LongValue.MARSHALLER, Longs.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, double... value) {
+    public Builder setArgument(String name, double... value) {
       nameArgs.put(name, toArgument(name, DoubleValue.MARSHALLER, Doubles.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, boolean... value) {
+    public Builder setArgument(String name, boolean... value) {
       nameArgs.put(name, toArgument(name, BooleanValue.MARSHALLER, Booleans.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, DateTime... value) {
+    public Builder setArgument(String name, DateTime... value) {
       nameArgs.put(name, toArgument(name, DateTimeValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, Key... value) {
+    public Builder setArgument(String name, Key... value) {
       nameArgs.put(name, toArgument(name, KeyValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, PartialEntity... value) {
+    public Builder setArgument(String name, PartialEntity... value) {
       nameArgs.put(name, toArgument(name, EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, Blob... value) {
+    public Builder setArgument(String name, Blob... value) {
       nameArgs.put(name, toArgument(name, BlobValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(Cursor cursor) {
+    public Builder addArgument(Cursor cursor) {
       numberArgs.add(new Argument(null, cursor));
       return this;
     }
 
-    public Builder addArgument(String... value) {
+    public Builder addArgument(String... value) {
       numberArgs.add(toArgument(StringValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(long... value) {
+    public Builder addArgument(long... value) {
       numberArgs.add(toArgument(LongValue.MARSHALLER, Longs.asList(value)));
       return this;
     }
 
-    public Builder addArgument(double... value) {
+    public Builder addArgument(double... value) {
       numberArgs.add(toArgument(DoubleValue.MARSHALLER, Doubles.asList(value)));
       return this;
     }
 
-    public Builder addArgument(boolean... value) {
+    public Builder addArgument(boolean... value) {
       numberArgs.add(toArgument(BooleanValue.MARSHALLER, Booleans.asList(value)));
       return this;
     }
 
-    public Builder addArgument(DateTime... value) {
+    public Builder addArgument(DateTime... value) {
       numberArgs.add(toArgument(DateTimeValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(Key... value) {
+    public Builder addArgument(Key... value) {
       numberArgs.add(toArgument(KeyValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(PartialEntity... value) {
+    public Builder addArgument(PartialEntity... value) {
       numberArgs.add(toArgument(EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(Blob... value) {
+    public Builder addArgument(Blob... value) {
       numberArgs.add(toArgument(BlobValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public GqlQuery build() {
+    public GqlQuery build() {
       return new GqlQuery<>(this);
     }
 
@@ -267,7 +291,7 @@ private static Argument toArgument(String name, Value.BuilderFactory builderFact
     }
   }
 
-  private GqlQuery(Builder builder) {
+  private GqlQuery(Builder builder) {
     super(builder.resultClass, builder.namespace);
     queryString = builder.queryString;
     allowLiteral = builder.allowLiteral;
@@ -346,35 +370,20 @@ protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
   }
 
   @Override
-  protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
-    throw new UnsupportedOperationException("paging not implemented yet");
-    /*
-    // TODO: THIS IS A MAJOR HACK, remove when possible. see b/18705483
-    String PREFIX_GROUP = "\\s*(?SELECT .*?)";
-    String OFFSET_GROUP =
-        "(\\s+OFFSET\\s+(?[^\\s]+)(\\s+\\+\\s+(?[^\\s]+))?)?";
-    String LIMIT_GROUP =
-        "(\\s+LIMIT\\s+((?[^\\s]+)|FIRST \\((?[^,]+,[^\\)]+)\\)))?\\s*";
-    Pattern pattern =
-        Pattern.compile(PREFIX_GROUP + OFFSET_GROUP + LIMIT_GROUP, Pattern.CASE_INSENSITIVE);
-
-    Matcher matcher = pattern.matcher(queryString);
-
-    if (!matcher.matches()) {
-      throw new UnsupportedOperationException("paging for this query is not implemented yet");
-    }
-    */
+  protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
+    // See b/18705483
+    throw new UnsupportedOperationException("paging for this query is not implemented yet");
   }
 
   @Override
-  protected Object fromPb(ResultClass resultType, String namespace, byte[] bytesPb)
+  protected Object fromPb(ResultClass resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
     return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
   }
 
-  static  GqlQuery fromPb(ResultClass resultType, String namespace,
+  static  GqlQuery fromPb(ResultClass resultType, String namespace,
       DatastoreV1.GqlQuery queryPb) {
-    Builder builder = new Builder<>(resultType, queryPb.getQueryString());
+    Builder builder = new Builder<>(resultType, queryPb.getQueryString());
     builder.namespace(namespace);
     if (queryPb.hasAllowLiteral()) {
       builder.allowLiteral = queryPb.getAllowLiteral();
@@ -404,7 +413,7 @@ public static GqlQuery.Builder builder(String gql) {
    *
    * @see GQL Reference
    */
-  public static  GqlQuery.Builder builder(ResultClass resultClass, String gql) {
+  public static  GqlQuery.Builder builder(ResultClass resultClass, String gql) {
     return new GqlQuery.Builder<>(resultClass, gql);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index 41cabfccb0ff..b1c3dac6e970 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -47,7 +47,11 @@ protected PartialEntity(PartialKey key, ImmutableSortedMap> pro
     this.key = key;
   }
 
-  public Entity newEntity(Key key) {
+  /**
+   * Returns a new {@link #Entity} with the same properties as this one and
+   * with the given {@code key}.
+   */
+  public Entity toEntity(Key key) {
     return new Entity(key, ImmutableSortedMap.>copyOf(properties()));
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 4959289e67ec..03df75c00312 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -9,21 +9,21 @@
 import com.google.protobuf.InvalidProtocolBufferException;
 
 
-// TODO(ozarov): add a usage examples (gql and regular)
 /**
  * A Google Cloud Datastore query.
+ * For usage examples see {@link GqlQuery} and {@link StructuredQuery}.
  *
- * @param  the type of the values returned by this query.
+ * @param  the type of the values returned by this query.
  * @see Datastore Queries
  */
-public abstract class Query extends Serializable {
+public abstract class Query extends Serializable {
 
   private static final long serialVersionUID = -2748141759901313101L;
 
-  private final ResultClass resultClass;
+  private final ResultClass resultClass;
   private final String namespace;
 
-  public static class ResultClass implements java.io.Serializable {
+  public static class ResultClass implements java.io.Serializable {
 
     private static final long serialVersionUID = 2104157695425806623L;
     private static final ResultClass UNKNOWN = new ResultClass<>(Object.class);
@@ -32,13 +32,13 @@ public static class ResultClass implements java.io.Serializable {
     private static final ResultClass PROJECTION =
         new ResultClass<>(PartialEntity.class);
 
-    private final Class value;
+    private final Class value;
 
-    private ResultClass(Class value) {
+    private ResultClass(Class value) {
       this.value = checkNotNull(value);
     }
 
-    public Class value() {
+    public Class value() {
       return value;
     }
 
@@ -131,22 +131,22 @@ Key convert(DatastoreV1.Entity value) {
       }
 
       @Override
-      ResultClass resultClass() {
-        return ResultClass.projection();
+      ResultClass resultClass() {
+        return ResultClass.keyOnly();
       }
     };
 
-    abstract  T convert(DatastoreV1.Entity value);
+    abstract  V convert(DatastoreV1.Entity value);
 
     abstract ResultClass resultClass();
   }
 
-  Query(ResultClass resultClass, String namespace) {
+  Query(ResultClass resultClass, String namespace) {
     this.resultClass = checkNotNull(resultClass);
     this.namespace = namespace;
   }
 
-  ResultClass getResultClass() {
+  ResultClass resultClass() {
     return resultClass;
   }
 
@@ -167,10 +167,10 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(resultClass, namespace, bytesPb);
   }
 
-  protected abstract Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
+  protected abstract Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException;
 
   protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb);
 
-  protected abstract Query nextQuery(DatastoreV1.QueryResultBatch responsePb);
+  protected abstract Query nextQuery(DatastoreV1.QueryResultBatch responsePb);
 }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index c569fc7bd361..4c69f6b73eaa 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -9,7 +9,7 @@
  *
  * @param V the type of the results value.
  */
-public interface QueryResult extends Iterator {
+public interface QueryResult extends Iterator {
 
   /**
    * Returns the actual class of the result's values.
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index 3f81a7fd0010..eecf6a196e6c 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -14,11 +14,13 @@ class QueryResultImpl extends AbstractIterator implements QueryResult {
 
   private final DatastoreServiceImpl datastore;
   private final DatastoreV1.ReadOptions readOptionsPb;
+  private final DatastoreV1.PartitionId partitionIdPb;
   private final Query.ResultClass resultClass;
   private Query query;
   private Query.Type type;
   private DatastoreV1.QueryResultBatch resultPb;
   private Iterator entityResultPbIter;
+  //private ByteString cursor; // only available in v1beta3
 
   static {
     ImmutableMap.Builder builder =
@@ -34,7 +36,15 @@ class QueryResultImpl extends AbstractIterator implements QueryResult {
     this.datastore = datastore;
     this.readOptionsPb = readOptionsPb;
     this.query = query;
-    this.resultClass = query.getResultClass();
+    resultClass = query.resultClass();
+    DatastoreV1.PartitionId.Builder pbBuilder = DatastoreV1.PartitionId.newBuilder();
+    pbBuilder.setDatasetId(datastore.options().dataset());
+    if (query.namespace() != null) {
+      pbBuilder.setNamespace(query.namespace());
+    } else if (datastore.options().namespace() != null) {
+      pbBuilder.setNamespace(datastore.options().namespace());
+    }
+    partitionIdPb = pbBuilder.build();
     sendRequest();
   }
 
@@ -43,18 +53,11 @@ private DatastoreV1.QueryResultBatch sendRequest() {
     if (readOptionsPb != null) {
       requestPb.setReadOptions(readOptionsPb);
     }
-    DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder();
-    partitionIdPb.setDatasetId(datastore.options().dataset());
-    String namespace = query.namespace() != null
-        ? query.namespace()
-        : datastore.options().namespace();
-    if (namespace != null) {
-      partitionIdPb.setNamespace(namespace);
-    }
-    requestPb.setPartitionId(partitionIdPb.build());
+    requestPb.setPartitionId(partitionIdPb);
     query.populatePb(requestPb);
     resultPb = datastore.runQuery(requestPb.build()).getBatch();
     entityResultPbIter = resultPb.getEntityResultList().iterator();
+    // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
     type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
     Preconditions.checkState(resultClass.isAssignableFrom(type.resultClass()),
         "Unexpected result type");
@@ -68,9 +71,12 @@ protected T computeNext() {
       query = query.nextQuery(resultPb);
       sendRequest();
     }
-    return entityResultPbIter.hasNext()
-        ? type.convert(entityResultPbIter.next().getEntity())
-        : endOfData();
+    if (!entityResultPbIter.hasNext()) {
+      return endOfData();
+    }
+    DatastoreV1.EntityResult entityResultPb = entityResultPbIter.next();
+    //cursor = entityResultPb.getCursor(); // only available in v1beta3
+    return type.convert(entityResultPb.getEntity());
   }
 
   @Override
@@ -80,7 +86,7 @@ public Class resultClass() {
 
   @Override
   public Cursor cursor() {
-    // TODO(ozarov): implement when v1beta3 is available
+    //return new Cursor(cursor); // only available in v1beta3
     return null;
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index a4d92290e964..dc4a14727e2e 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -22,7 +22,7 @@
 import java.util.List;
 import java.util.Objects;
 
-public abstract class StructuredQuery extends Query {
+public class StructuredQuery extends Query {
 
   private static final long serialVersionUID = 546838955624019594L;
   private static final String KEY_PROPERTY_NAME = "__key__";
@@ -34,7 +34,7 @@ public abstract class StructuredQuery extends Query {
   private final transient ImmutableList orderBy;
   private final transient Cursor startCursor;
   private final transient Cursor endCursor;
-  private final transient Integer offset;
+  private final transient int offset;
   private final transient Integer limit;
 
   public abstract static class Filter implements Serializable {
@@ -84,6 +84,32 @@ private CompositeFilter(Operator operator, ImmutableList filters) {
       Preconditions.checkArgument(!filters.isEmpty(), "filters list must not be empty");
     }
 
+    @Override
+    public String toString() {
+      ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+      toStringHelper.add("operator", operator);
+      toStringHelper.add("filters", filters);
+      return toStringHelper.toString();
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(operator, filters);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof CompositeFilter)) {
+        return false;
+      }
+      CompositeFilter other = (CompositeFilter) obj;
+      return operator.equals(other.operator)
+          && filters.equals(other.filters);
+    }
+
     static CompositeFilter fromPb(DatastoreV1.CompositeFilter compositeFilterPb) {
       Operator operator = Operator.fromPb(compositeFilterPb.getOperator());
       ImmutableList.Builder filters = ImmutableList.builder();
@@ -147,6 +173,15 @@ public static PropertyFilter fromPb(DatastoreV1.PropertyFilter propertyFilterPb)
       return new PropertyFilter(property, operator, value);
     }
 
+    @Override
+    public String toString() {
+      ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+      toStringHelper.add("property", property);
+      toStringHelper.add("operator", operator);
+      toStringHelper.add("value", value);
+      return toStringHelper.toString();
+    }
+
     @Override
     public int hashCode() {
       return Objects.hash(property, operator, value);
@@ -506,9 +541,9 @@ public static Projection first(String property) {
     }
   }
 
-  static abstract class BaseBuilder> {
+  static class BaseBuilder> {
 
-    private ResultClass resultClass;
+    private ResultClass resultClass;
     private String namespace;
     private String kind;
     private List projection = new LinkedList<>();
@@ -517,10 +552,10 @@ static abstract class BaseBuilder> {
     private List orderBy = new LinkedList<>();
     private Cursor startCursor;
     private Cursor endCursor;
-    private Integer offset;
+    private int offset;
     private Integer limit;
 
-    BaseBuilder(ResultClass resultClass) {
+    BaseBuilder(ResultClass resultClass) {
       this.resultClass = resultClass;
     }
 
@@ -549,14 +584,14 @@ public B endCursor(Cursor endCursor) {
       return self();
     }
 
-    public B offset(Integer offset) {
-      Preconditions.checkArgument(offset == null || offset >= 0, "offset must not be negative");
+    public B offset(int offset) {
+      Preconditions.checkArgument(offset >= 0, "offset must not be negative");
       this.offset = offset;
       return self();
     }
 
     public B limit(Integer limit) {
-      Preconditions.checkArgument(limit == null || offset > 0, "limit must be positive");
+      Preconditions.checkArgument(limit == null || limit > 0, "limit must be positive");
       this.limit = limit;
       return self();
     }
@@ -654,41 +689,43 @@ protected B mergeFrom(DatastoreV1.Query queryPb) {
       return self();
     }
 
-    public abstract StructuredQuery build();
+    public StructuredQuery build() {
+      return new StructuredQuery<>(this);
+    }
   }
 
-  public static final class FullQueryBuilder extends BaseBuilder {
+  public static final class Builder extends BaseBuilder> {
 
-    FullQueryBuilder() {
-      super(ResultClass.full());
-    }
-
-    @Override
-    public FullQuery build() {
-      clearProjection();
-      clearGroupBy();
-      return new FullQuery(this);
+    public Builder(ResultClass resultClass) {
+      super(resultClass);
     }
   }
 
-  public static final class KeyOnlyQueryBuilder extends BaseBuilder {
+  public static final class KeyOnlyBuilder extends BaseBuilder {
 
-    public KeyOnlyQueryBuilder() {
+    public KeyOnlyBuilder() {
       super(ResultClass.keyOnly());
+      projection(Projection.property(KEY_PROPERTY_NAME));
     }
 
     @Override
-    public KeyOnlyQuery build() {
+    protected KeyOnlyBuilder mergeFrom(DatastoreV1.Query queryPb) {
+      super.mergeFrom(queryPb);
       projection(Projection.property(KEY_PROPERTY_NAME));
       clearGroupBy();
+      return this;
+    }
+
+    @Override
+    public KeyOnlyQuery build() {
       return new KeyOnlyQuery(this);
     }
   }
 
-  public static final class ProjectionQueryBuilder
-      extends BaseBuilder {
+  public static final class ProjectionBuilder
+      extends BaseBuilder {
 
-    public ProjectionQueryBuilder() {
+    public ProjectionBuilder() {
       super(ResultClass.projection());
     }
 
@@ -698,50 +735,41 @@ public ProjectionQuery build() {
     }
 
     @Override
-    public ProjectionQueryBuilder clearProjection() {
+    public ProjectionBuilder clearProjection() {
       return super.clearProjection();
     }
 
     @Override
-    public ProjectionQueryBuilder projection(Projection projection, Projection... others) {
+    public ProjectionBuilder projection(Projection projection, Projection... others) {
       return super.projection(projection, others);
     }
 
     @Override
-    public ProjectionQueryBuilder addProjection(Projection projection, Projection... others) {
+    public ProjectionBuilder addProjection(Projection projection, Projection... others) {
       return super.addProjection(projection, others);
     }
 
     @Override
-    public ProjectionQueryBuilder clearGroupBy() {
+    public ProjectionBuilder clearGroupBy() {
       return super.clearGroupBy();
     }
 
     @Override
-    public ProjectionQueryBuilder groupBy(String property, String... others) {
+    public ProjectionBuilder groupBy(String property, String... others) {
       return super.groupBy(property, others);
     }
 
     @Override
-    public ProjectionQueryBuilder addGroupBy(String property, String... others) {
+    public ProjectionBuilder addGroupBy(String property, String... others) {
       return super.addGroupBy(property, others);
     }
   }
 
-  public static final class FullQuery extends StructuredQuery {
-
-    private static final long serialVersionUID = -84461800292593840L;
-
-    FullQuery(FullQueryBuilder builder) {
-      super(builder);
-    }
-  }
-
   public static final class KeyOnlyQuery extends StructuredQuery {
 
     private static final long serialVersionUID = -7502917784216095473L;
 
-    KeyOnlyQuery(KeyOnlyQueryBuilder builder) {
+    KeyOnlyQuery(KeyOnlyBuilder builder) {
       super(builder);
     }
   }
@@ -750,7 +778,7 @@ public static final class ProjectionQuery extends StructuredQuery
 
     private static final long serialVersionUID = -3333183044486150649L;
 
-    ProjectionQuery(ProjectionQueryBuilder builder) {
+    ProjectionQuery(ProjectionBuilder builder) {
       super(builder);
       Preconditions.checkState(!keyOnly(),
           "Projection query can't project only '__key__', use KeyOnlyQuery instead.");
@@ -767,7 +795,7 @@ public List groupBy() {
     }
   }
 
-  StructuredQuery(BaseBuilder builder) {
+  StructuredQuery(BaseBuilder builder) {
     super(builder.resultClass, builder.namespace);
     kind = builder.kind;
     projection = ImmutableList.copyOf(builder.projection);
@@ -840,7 +868,7 @@ public Cursor endCursor() {
     return endCursor;
   }
 
-  public Integer offset() {
+  public int offset() {
     return offset;
   }
 
@@ -854,13 +882,23 @@ protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
   }
 
   @Override
-  protected StructuredQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
-    // TODO: implment
-    throw new UnsupportedOperationException("paging not implemented yet");
+  protected StructuredQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
+    Builder builder = new Builder<>(resultClass());
+    builder.mergeFrom(toPb());
+    builder.startCursor(new Cursor(responsePb.getEndCursor()));
+    if (offset > 0 && responsePb.getSkippedResults() < offset) {
+      builder.offset(offset - responsePb.getSkippedResults());
+    } else {
+      builder.offset(0);
+      if (limit != null) {
+        builder.limit(limit - responsePb.getEntityResultCount());
+      }
+    }
+    return builder.build();
   }
 
   @Override
-  protected Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
+  protected Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
     return fromPb(resultClass, namespace, DatastoreV1.Query.parseFrom(bytesPb));
   }
@@ -877,7 +915,7 @@ protected DatastoreV1.Query toPb() {
     if (endCursor != null) {
       queryPb.setEndCursor(endCursor.byteString());
     }
-    if (offset != null) {
+    if (offset > 0) {
       queryPb.setOffset(offset);
     }
     if (limit != null) {
@@ -902,24 +940,24 @@ static StructuredQuery fromPb(ResultClass resultClass, String namespace,
       DatastoreV1.Query queryPb) {
     BaseBuilder builder;
     if (resultClass.equals(ResultClass.full())) {
-      builder = new FullQueryBuilder();
+      builder = builder();
     } else if (resultClass.equals(ResultClass.keyOnly())) {
-      builder = new KeyOnlyQueryBuilder();
+      builder = keyOnlyBuilder();
     } else {
-      builder = new ProjectionQueryBuilder();
+      builder = projectionBuilder();
     }
     return builder.namespace(namespace).mergeFrom(queryPb).build();
   }
 
-  public static FullQueryBuilder builder() {
-    return new FullQueryBuilder();
+  public static Builder builder() {
+    return new Builder<>(ResultClass.full());
   }
 
-  public static KeyOnlyQueryBuilder keyOnlyBuilder() {
-    return new KeyOnlyQueryBuilder();
+  public static KeyOnlyBuilder keyOnlyBuilder() {
+    return new KeyOnlyBuilder();
   }
 
-  public static ProjectionQueryBuilder projectionBuilder() {
-    return new ProjectionQueryBuilder();
+  public static ProjectionBuilder projectionBuilder() {
+    return new ProjectionBuilder();
   }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 60525f39f894..b46dee10bde0 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -8,6 +8,9 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.gcloud.datastore.Query.ResultClass;
+import com.google.gcloud.datastore.StructuredQuery.OrderBy;
+
 import org.junit.Before;
 import org.junit.Test;
 
@@ -242,17 +245,75 @@ public void testNewBatchWriter() {
 
   @Test
   public void testRunGqlQueryNoCasting() {
-    fail("Not yet implemented");
+    Query query1 = GqlQuery.builder(ResultClass.full(), "select * from " + KIND1).build();
+    QueryResult results1 = datastore.runQuery(query1);
+    assertTrue(results1.hasNext());
+    assertEquals(ENTITY1, results1.next());
+    assertFalse(results1.hasNext());
+
+    datastore.put(ENTITY3);
+    Query query2 =  GqlQuery.builder(
+        ResultClass.full(), "select * from " + KIND2 + " order by __key__").build();
+    QueryResult results2 = datastore.runQuery(query2);
+    assertTrue(results2.hasNext());
+    assertEquals(ENTITY2, results2.next());
+    assertTrue(results2.hasNext());
+    assertEquals(ENTITY3, results2.next());
+    assertFalse(results2.hasNext());
+
+    query1 = GqlQuery.builder(ResultClass.full(), "select * from bla").build();
+    results1 = datastore.runQuery(query1);
+    assertFalse(results1.hasNext());
+
+    Query keyOnlyQuery =
+        GqlQuery.builder(ResultClass.keyOnly(), "select __key__ from " + KIND1).build();
+    QueryResult keyOnlyResults = datastore.runQuery(keyOnlyQuery);
+    assertTrue(keyOnlyResults.hasNext());
+    assertEquals(KEY1, keyOnlyResults.next());
+    assertFalse(keyOnlyResults.hasNext());
+
+    Query projectionQuery = GqlQuery.builder(
+        ResultClass.projection(), "select str from " + KIND1).build();
+    QueryResult projectionResult = datastore.runQuery(projectionQuery);
+    assertTrue(projectionResult.hasNext());
+    PartialEntity partialEntity = projectionResult.next();
+    assertEquals("str", partialEntity.getString("str"));
+    assertEquals(1, partialEntity.names().size());
+    assertFalse(projectionResult.hasNext());
   }
 
   @Test
   public void testRunGqlQueryWithCasting() {
-    fail("Not yet implemented");
+    @SuppressWarnings("unchecked")
+    Query query1 = (Query) GqlQuery.builder("select * from " + KIND1).build();
+    QueryResult results1 = datastore.runQuery(query1);
+    assertTrue(results1.hasNext());
+    assertEquals(ENTITY1, results1.next());
+    assertFalse(results1.hasNext());
+
+    Query query2 = GqlQuery.builder("select * from " + KIND1).build();
+    QueryResult results2 = datastore.runQuery(query2);
+    assertEquals(Entity.class, results2.resultClass());
+    @SuppressWarnings("unchecked")
+    QueryResult results3 = (QueryResult) results2;
+    assertTrue(results3.hasNext());
+    assertEquals(ENTITY1, results3.next());
+    assertFalse(results3.hasNext());
   }
 
   @Test
-  public void testRunStructuredQueryFull() {
-    fail("Not yet implemented");
+  public void testRunStructuredQuery() {
+    StructuredQuery query =
+        StructuredQuery.builder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
+    QueryResult results1 = datastore.runQuery(query);
+    assertTrue(results1.hasNext());
+    assertEquals(ENTITY1, results1.next());
+    assertFalse(results1.hasNext());
+
+    StructuredQuery keyOnlyQuery =  StructuredQuery.keyOnlyBuilder().kind(KIND1).build();
+
+
+    // todo(ozarov): construct a test to very nextQuery/pagination
   }
 
   @Test
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 79c08cffa674..f66bb6bc319a 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -8,7 +8,10 @@
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Multimap;
 import com.google.gcloud.datastore.Query.ResultClass;
+import com.google.gcloud.datastore.StructuredQuery.CompositeFilter;
+import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
+import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
 import com.google.gcloud.datastore.Value.Type;
 
 import org.junit.Test;
@@ -29,6 +32,7 @@ public class SerializationTest {
   private static final DateTime DATE_TIME1 = DateTime.now();
   private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world"));
   private static final Cursor CURSOR1 = Cursor.copyFrom(new byte[] {1,2});
+  private static final Cursor CURSOR2 = Cursor.copyFrom(new byte[] {10});
   private static final Query GQL1 =
       GqlQuery.builder("select * from kind1 where name = @name and age > @1")
       .setArgument("name", "name1")
@@ -42,10 +46,22 @@ public class SerializationTest {
       .namespace("ns1")
       .build();
   private static final Query QUERY1 = StructuredQuery.builder().kind("kind1").build();
-  private static final Query QUERY2 = StructuredQuery.keyOnlyBuilder().kind("k").filter(
-      StructuredQuery.PropertyFilter.eq("p1", "hello")).build();
-  private static final Query QUERY3 =
-      StructuredQuery.projectionBuilder().kind("k").projection(Projection.property("p")).build();
+  private static final Query QUERY2 = StructuredQuery.keyOnlyBuilder()
+      .kind("k")
+      .filter(PropertyFilter.eq("p1", "hello"))
+      .build();
+  private static final Query QUERY3 = StructuredQuery.projectionBuilder()
+      .kind("k")
+      .namespace("ns1")
+      .projection(Projection.property("p"))
+      .limit(100)
+      .offset(5)
+      .startCursor(CURSOR1)
+      .endCursor(CURSOR2)
+      .filter(CompositeFilter.and(PropertyFilter.gt("p1", 10), PropertyFilter.eq("a", "v")))
+      .addGroupBy("p")
+      .addOrderBy(OrderBy.asc("p"))
+      .build();
   private static final KeyValue KEY_VALUE = KeyValue.of(KEY1);
   private static final NullValue NULL_VALUE = NullValue.builder().indexed(true).build();
   private static final StringValue STRING_VALUE = StringValue.of("hello");

From 86625f83b117123a1ef8d793796910029a0842f0 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Mon, 15 Dec 2014 21:41:07 -0800
Subject: [PATCH 057/732] add more tests and javadoc for queries

---
 .../google/gcloud/datastore/BaseEntity.java   | 10 ++--
 .../com/google/gcloud/datastore/GqlQuery.java |  6 +-
 .../gcloud/datastore/StructuredQuery.java     | 50 +++++++++++++---
 .../datastore/DatastoreServiceTest.java       | 59 +++++++++++++++----
 4 files changed, 98 insertions(+), 27 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 606630a5b10c..6d5784839e94 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -68,11 +68,6 @@ public B set(String name, Value value) {
       return self();
     }
 
-    public B setNull(String name) {
-      properties.put(name, of());
-      return self();
-    }
-
     public B set(String name, String value) {
       properties.put(name, of(value));
       return self();
@@ -123,6 +118,11 @@ public B set(String name, Blob value) {
       return self();
     }
 
+    public B setNull(String name) {
+      properties.put(name, of());
+      return self();
+    }
+
     public E build() {
       return build(ImmutableSortedMap.copyOf(properties));
     }
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index ff8633760f87..f8769d8ac0ae 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -26,7 +26,7 @@
  *
  * 
A usage example:

  *
- * When the type of the results is known the preferred usage would be:
+ * 
When the type of the results is known the preferred usage would be:
  * 
 {@code
  *   Query query = GqlQuery.builder(ResultClass.full(), "select * from kind").build();
  *   QueryResult results = datastore.runQuery(query);
@@ -36,7 +36,7 @@
  *   }
  * } 

  *
- * When the type of the results is unknown you can use this approach:
+ * 
When the type of the results is unknown you can use this approach:
  * 
 {@code
  *   Query query = GqlQuery.builder("select __key__ from kind").build();
  *   QueryResult results = datastore.runQuery(query);
@@ -48,6 +48,8 @@
  *     }
  *   }
  * } 

+ *
+ * @param  the type of the result values this query will produce
  * @see GQL Reference
  */
 public final class GqlQuery extends Query {
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index dc4a14727e2e..f9a733d1a529 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -22,6 +22,40 @@
 import java.util.List;
 import java.util.Objects;
 
+/**
+ * An implementation of a Google Cloud Datastore Query that can be constructed by providing
+ * all the specific query elements.
+ *
+ * 
A usage example:

+ *
+ * 
A simple query that returns all entities for a specific kind
+ * 
 {@code
+ *   StructuredQuery query = StructuredQuery.builder().kind(kind).build();
+ *   QueryResult results = datastore.runQuery(query);
+ *   while (results.hasNext()) {
+ *     Entity entity = results.next();
+ *     ...
+ *   }
+ * } 

+ *
+ * 
A less trivial example of a projection query that returns the first 10 results
+ * of "age" and "name" properties (sorted and grouped by "age") with an age greater than 18
+ * 
 {@code
+ *   StructuredQuery query = StructuredQuery.projectionBuilder()
+ *       .kind(kind)
+ *       .projection(Projection.property("age"), Projection.first("name"))
+ *       .filter(PropertyFilter.gt("age", 18))
+ *       .groupBy("age")
+ *       .orderBy(OrderBy.asc("age"))
+ *       .limit(10)
+ *       .build();
+ *   QueryResult results = datastore.runQuery(query);
+ *   ...
+ * } 

+ *
+ * @param  the type of the result values this query will produce
+ * @see Datastore queries
+ */
 public class StructuredQuery extends Query {
 
   private static final long serialVersionUID = 546838955624019594L;
@@ -361,8 +395,8 @@ public static PropertyFilter eq(String property, Blob value) {
       return new PropertyFilter(property, Operator.EQUAL, of(value));
     }
 
-    public static PropertyFilter hasAncestor(String property, Key key) {
-      return new PropertyFilter(property, Operator.HAS_ANCESTOR, of(key));
+    public static PropertyFilter hasAncestor(Key key) {
+      return new PropertyFilter(KEY_PROPERTY_NAME, Operator.HAS_ANCESTOR, of(key));
     }
 
     public static PropertyFilter isNull(String property) {
@@ -897,12 +931,6 @@ protected StructuredQuery nextQuery(DatastoreV1.QueryResultBatch responsePb)
     return builder.build();
   }
 
-  @Override
-  protected Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
-      throws InvalidProtocolBufferException {
-    return fromPb(resultClass, namespace, DatastoreV1.Query.parseFrom(bytesPb));
-  }
-
   @Override
   protected DatastoreV1.Query toPb() {
     DatastoreV1.Query.Builder queryPb = DatastoreV1.Query.newBuilder();
@@ -936,6 +964,12 @@ protected DatastoreV1.Query toPb() {
     return queryPb.build();
   }
 
+  @Override
+  protected Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
+      throws InvalidProtocolBufferException {
+    return fromPb(resultClass, namespace, DatastoreV1.Query.parseFrom(bytesPb));
+  }
+
   static StructuredQuery fromPb(ResultClass resultClass, String namespace,
       DatastoreV1.Query queryPb) {
     BaseBuilder builder;
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index b46dee10bde0..e7584afa7a83 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -10,6 +10,8 @@
 
 import com.google.gcloud.datastore.Query.ResultClass;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
+import com.google.gcloud.datastore.StructuredQuery.Projection;
+import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -48,7 +50,7 @@ public class DatastoreServiceTest {
       .set("bool", BOOL_VALUE).set("partial1", EntityValue.of(PARTIAL_ENTITY1))
       .set("list", LIST_VALUE2).build();
   private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
-      .setNull("null").build();
+      .set("name", "koko").setNull("null").set("age", 20).build();
   private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
       .set("null", NULL_VALUE).set("partial1", PARTIAL_ENTITY2).set("partial2", ENTITY2).build();
 
@@ -138,7 +140,29 @@ public void testTransactionWithRead() {
 
   @Test
   public void testTransactionWithQuery() {
-    fail("not implemented");
+    Query query =
+        StructuredQuery.builder().kind(KIND2).filter(PropertyFilter.hasAncestor(KEY2)).build();
+    Transaction transaction = datastore.newTransaction();
+    QueryResult results = transaction.runQuery(query);
+    assertEquals(ENTITY2, results.next());
+    assertFalse(results.hasNext());
+    transaction.add(ENTITY3);
+    transaction.commit();
+    assertEquals(ENTITY3, datastore.get(KEY3));
+
+    transaction = datastore.newTransaction();
+    results = transaction.runQuery(query);
+    assertEquals(ENTITY2, results.next());
+    transaction.delete(ENTITY3.key());
+    // update entity2 during the transaction
+    datastore.put(Entity.builder(ENTITY2).clear().build());
+    try {
+      transaction.commit();
+      fail("Expecting a failure");
+    } catch (DatastoreServiceException expected) {
+      expected.printStackTrace();
+      assertEquals(DatastoreServiceException.Code.ABORTED, expected.code());
+    }
   }
 
   @Test
@@ -311,19 +335,30 @@ public void testRunStructuredQuery() {
     assertFalse(results1.hasNext());
 
     StructuredQuery keyOnlyQuery =  StructuredQuery.keyOnlyBuilder().kind(KIND1).build();
+    QueryResult results2 = datastore.runQuery(keyOnlyQuery);
+    assertTrue(results2.hasNext());
+    assertEquals(ENTITY1.key(), results2.next());
+    assertFalse(results2.hasNext());
 
+    StructuredQuery projectionQuery = StructuredQuery.projectionBuilder()
+        .kind(KIND2)
+        .projection(Projection.property("age"), Projection.first("name"))
+        .filter(PropertyFilter.gt("age", 18))
+        .groupBy("age")
+        .orderBy(OrderBy.asc("age"))
+        .limit(10)
+        .build();
 
-    // todo(ozarov): construct a test to very nextQuery/pagination
-  }
-
-  @Test
-  public void testRunStructuredQueryProjection() {
-    fail("Not yet implemented");
-  }
+    QueryResult results3 = datastore.runQuery(projectionQuery);
+    assertTrue(results3.hasNext());
+    PartialEntity entity = results3.next();
+    assertEquals(ENTITY2.key(), entity.key());
+    assertEquals(20, entity.getLong("age"));
+    assertEquals("koko", entity.getString("name"));
+    assertEquals(2, entity.properties().size());
+    assertFalse(results3.hasNext());
 
-  @Test
-  public void testRunStructuredQueryKeysOnly() {
-    fail("Not yet implemented");
+    // TODO(ozarov): construct a test to very nextQuery/pagination
   }
 
   @Test

From bcb9a5981650638fa93ca68c9142d4996b31926c Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 16 Dec 2014 08:46:18 -0800
Subject: [PATCH 058/732] minor comments update

---
 .../com/google/gcloud/datastore/DatastoreServiceTest.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index e7584afa7a83..fadd24147c67 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -260,7 +260,7 @@ public void testNewBatchWriter() {
     assertNull(entities.next());
     assertFalse(entities.hasNext());
 
-    // TODO need to cover the cases of:
+    // TODO need to cover the following use-cases:
     // delete after put/add/update
     // put after delete/add/update
     // update after delete/add/put
@@ -358,7 +358,7 @@ public void testRunStructuredQuery() {
     assertEquals(2, entity.properties().size());
     assertFalse(results3.hasNext());
 
-    // TODO(ozarov): construct a test to very nextQuery/pagination
+    // TODO(ozarov): construct a test to verify nextQuery/pagination
   }
 
   @Test

From 26a1d81c459c2e0e7320f4ef83756476f9c04bb6 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 16 Dec 2014 17:37:33 -0800
Subject: [PATCH 059/732] 1) minor changes to datastore. 2) migrate exception
 handler from AE gcs client 3) start with gcs client.

---
 pom.xml                                       | 272 ++++++++++++++++--
 .../com/google/gcloud/ExceptionHandler.java   | 197 +++++++++++++
 .../java/com/google/gcloud/RetryHelper.java   | 228 +++++++++++++++
 .../java/com/google/gcloud/RetryParams.java   | 272 ++++++++++++++++++
 .../com/google/gcloud/ServiceOptions.java     |   2 +
 .../datastore/DatastoreServiceOptions.java    |   9 +-
 .../google/gcloud/datastore/package-info.java |   5 +-
 .../java/com/google/gcloud/storage/Acl.java   |   5 +
 .../com/google/gcloud/storage/Bucket.java     |  35 +++
 .../google/gcloud/storage/StorageService.java |   9 +
 .../gcloud/storage/StorageServiceFactory.java |  19 ++
 .../gcloud/storage/StorageServiceImpl.java    |  28 ++
 .../gcloud/storage/StorageServiceOptions.java | 100 +++++++
 .../google/gcloud/storage/package-info.java   |   6 +
 .../google/gcloud/ExceptionHandlerTest.java   | 113 ++++++++
 .../com/google/gcloud/RetryParamsTest.java    |  91 ++++++
 16 files changed, 1369 insertions(+), 22 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/ExceptionHandler.java
 create mode 100644 src/main/java/com/google/gcloud/RetryHelper.java
 create mode 100644 src/main/java/com/google/gcloud/RetryParams.java
 create mode 100644 src/main/java/com/google/gcloud/storage/Acl.java
 create mode 100644 src/main/java/com/google/gcloud/storage/Bucket.java
 create mode 100644 src/main/java/com/google/gcloud/storage/StorageService.java
 create mode 100644 src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
 create mode 100644 src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
 create mode 100644 src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
 create mode 100644 src/main/java/com/google/gcloud/storage/package-info.java
 create mode 100644 src/test/java/com/google/gcloud/ExceptionHandlerTest.java
 create mode 100644 src/test/java/com/google/gcloud/RetryParamsTest.java

diff --git a/pom.xml b/pom.xml
index 3a573d91459f..585f21738fa5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,4 +1,6 @@
-
+
+
   4.0.0
   com.ozarov.testing
   git-demo
@@ -7,59 +9,295 @@
     
       com.google.http-client
       google-http-client
-      1.18.0-rc
+      1.19.0
+      compile
     
     
       com.google.oauth-client
       google-oauth-client
-      1.18.0-rc
+      1.19.0
+      compile
     
     
       com.google.guava
       guava
       RELEASE
+      compile
     
     
       com.google.apis
-      
-        google-api-services-datastore-protobuf
-      
+      google-api-services-datastore-protobuf
       v1beta2-rev1-2.1.0
+      compile
     
     
       com.google.api-client
       google-api-client-appengine
-      1.18.0-rc
+      1.19.0
+      compile
     
     
       junit
       junit
+      RELEASE
       test
+    
+    
+      joda-time
+      joda-time
       RELEASE
+      compile
     
     
-    	joda-time
-    	joda-time
-    	RELEASE
+      org.json
+      json
+      20090211
+      compile
     
     
-    	org.json
-    	json
-    	20090211
+      com.google.apis
+      google-api-services-storage
+      v1-rev23-1.19.0
+      compile
+    
+    
+      com.google.apis
+      google-api-services-datastore
+      v1beta2-rev23-1.19.0
     
   
+  
+    
+      
+        false
+      
+      central
+      Central Repository
+      http://repo.maven.apache.org/maven2
+    
+  
+  
+    
+      
+        never
+      
+      
+        false
+      
+      central
+      Central Repository
+      http://repo.maven.apache.org/maven2
+    
+  
   
+    /usr/local/google/home/ozarov/git/git-demo/src/main/java
+    /usr/local/google/home/ozarov/git/git-demo/src/main/scripts
+    /usr/local/google/home/ozarov/git/git-demo/src/test/java
+    /usr/local/google/home/ozarov/git/git-demo/target/classes
+    /usr/local/google/home/ozarov/git/git-demo/target/test-classes
+    
+      
+        /usr/local/google/home/ozarov/git/git-demo/src/main/resources
+      
+    
+    
+      
+        /usr/local/google/home/ozarov/git/git-demo/src/test/resources
+      
+    
+    /usr/local/google/home/ozarov/git/git-demo/target
+    git-demo-0.0.1-SNAPSHOT
+    
+      
+        
+          maven-antrun-plugin
+          1.3
+        
+        
+          maven-assembly-plugin
+          2.2-beta-5
+        
+        
+          maven-dependency-plugin
+          2.8
+        
+        
+          maven-release-plugin
+          2.3.2
+        
+      
+    
     
       
-        org.apache.maven.plugins
         maven-compiler-plugin
         3.1
+        
+          
+            default-testCompile
+            test-compile
+            
+              testCompile
+            
+            
+              1.7
+              1.7
+              UTF-8
+            
+          
+          
+            default-compile
+            compile
+            
+              compile
+            
+            
+              1.7
+              1.7
+              UTF-8
+            
+          
+        
         
-                1.7
-                1.7
-                UTF-8
+          1.7
+          1.7
+          UTF-8
+        
+      
+      
+        maven-clean-plugin
+        2.5
+        
+          
+            default-clean
+            clean
+            
+              clean
+            
+          
+        
+      
+      
+        maven-install-plugin
+        2.4
+        
+          
+            default-install
+            install
+            
+              install
+            
+          
+        
+      
+      
+        maven-resources-plugin
+        2.6
+        
+          
+            default-resources
+            process-resources
+            
+              resources
+            
+          
+          
+            default-testResources
+            process-test-resources
+            
+              testResources
+            
+          
+        
+      
+      
+        maven-surefire-plugin
+        2.12.4
+        
+          
+            default-test
+            test
+            
+              test
+            
+          
+        
+      
+      
+        maven-jar-plugin
+        2.4
+        
+          
+            default-jar
+            package
+            
+              jar
+            
+          
+        
+      
+      
+        maven-deploy-plugin
+        2.7
+        
+          
+            default-deploy
+            deploy
+            
+              deploy
+            
+          
+        
+      
+      
+        maven-site-plugin
+        3.3
+        
+          
+            default-site
+            site
+            
+              site
+            
+            
+              /usr/local/google/home/ozarov/git/git-demo/target/site
+              
+                
+                  org.apache.maven.plugins
+                  maven-project-info-reports-plugin
+                
+              
+            
+          
+          
+            default-deploy
+            site-deploy
+            
+              deploy
+            
+            
+              /usr/local/google/home/ozarov/git/git-demo/target/site
+              
+                
+                  org.apache.maven.plugins
+                  maven-project-info-reports-plugin
+                
+              
             
+          
+        
+        
+          /usr/local/google/home/ozarov/git/git-demo/target/site
+          
+            
+              org.apache.maven.plugins
+              maven-project-info-reports-plugin
+            
+          
+        
       
     
   
+  
+    /usr/local/google/home/ozarov/git/git-demo/target/site
+  
 
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
new file mode 100644
index 000000000000..7a7371e175ab
--- /dev/null
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -0,0 +1,197 @@
+package com.google.gcloud;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+import java.io.Serializable;
+import java.lang.reflect.Method;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+/**
+ * Exception handling used by {@link RetryHelper}.
+ */
+public final class ExceptionHandler implements Serializable {
+
+  private static final long serialVersionUID = -2460707015779532919L;
+
+  private static final ExceptionHandler DEFAULT_INSTANCE =
+      builder().retryOn(Exception.class).abortOn(RuntimeException.class).build();
+
+  private final ImmutableSet> retriableExceptions;
+  private final ImmutableSet> nonRetriableExceptions;
+  private final Set retryInfos = Sets.newHashSet();
+
+  /**
+   * ExceptionHandler builder.
+   */
+  public static class Builder {
+
+    private final ImmutableSet.Builder> retriableExceptions =
+        new ImmutableSet.Builder<>();
+    private final ImmutableSet.Builder> nonRetriableExceptions =
+        new ImmutableSet.Builder<>();
+
+    private Builder() {
+    }
+
+    /**
+     * Specify on what exceptions to continue.
+     *
+     * @param exceptions retry should continue when such exceptions are thrown
+     * @return the Builder for chaining
+     */
+    @SafeVarargs
+    public final Builder retryOn(Class... exceptions) {
+      for (Class exception : exceptions) {
+        retriableExceptions.add(checkNotNull(exception));
+      }
+      return this;
+    }
+
+    /**
+     * Specify on what exceptions to abort.
+     *
+     * @param exceptions retry should abort when such exceptions are thrown
+     * @return the Builder for chaining
+     */
+    @SafeVarargs
+    public final Builder abortOn(Class... exceptions) {
+      for (Class exception : exceptions) {
+        nonRetriableExceptions.add(checkNotNull(exception));
+      }
+      return this;
+    }
+
+    /**
+     * Returns a new ExceptionHandler instance.
+     */
+    public ExceptionHandler build() {
+      return new ExceptionHandler(this);
+    }
+  }
+
+  @VisibleForTesting
+  static final class RetryInfo implements Serializable {
+
+    private static final long serialVersionUID = -4264634837841455974L;
+    private final Class exception;
+    private final boolean retry;
+    private final Set children = Sets.newHashSet();
+
+    RetryInfo(Class exception, boolean retry) {
+      this.exception = checkNotNull(exception);
+      this.retry = retry;
+    }
+
+    @Override
+    public int hashCode() {
+      return exception.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == this) {
+        return true;
+      }
+      if (!(obj instanceof RetryInfo)) {
+        return false;
+      }
+      // We only care about exception in equality as we allow only one instance per exception
+      return ((RetryInfo) obj).exception.equals(exception);
+    }
+  }
+
+  private ExceptionHandler(Builder builder) {
+    retriableExceptions = builder.retriableExceptions.build();
+    nonRetriableExceptions = builder.nonRetriableExceptions.build();
+    Preconditions.checkArgument(
+        Sets.intersection(retriableExceptions, nonRetriableExceptions).isEmpty(),
+        "Same exception was found in both retriable and non-retriable sets");
+    for (Class exception : retriableExceptions) {
+      addToRetryInfos(retryInfos, new RetryInfo(exception, true));
+    }
+    for (Class exception : nonRetriableExceptions) {
+      addToRetryInfos(retryInfos,  new RetryInfo(exception, false));
+    }
+  }
+
+  private static void addToRetryInfos(Set retryInfos, RetryInfo retryInfo) {
+    for (RetryInfo current : retryInfos) {
+      if (current.exception.isAssignableFrom(retryInfo.exception)) {
+        addToRetryInfos(current.children, retryInfo);
+        return;
+      }
+      if (retryInfo.exception.isAssignableFrom(current.exception)) {
+        retryInfo.children.add(current);
+      }
+    }
+    retryInfos.removeAll(retryInfo.children);
+    retryInfos.add(retryInfo);
+  }
+
+
+  private static RetryInfo findMostSpecificRetryInfo(Set retryInfos,
+      Class exception) {
+    for (RetryInfo current : retryInfos) {
+      if (current.exception.isAssignableFrom(exception)) {
+        RetryInfo  match = findMostSpecificRetryInfo(current.children, exception);
+        return match == null ? current : match;
+      }
+    }
+    return null;
+  }
+
+  // called for Class, therefore a "call" method must be found.
+  private static Method getCallableMethod(Class clazz) {
+    try {
+      return clazz.getDeclaredMethod("call");
+    } catch (NoSuchMethodException e) {
+      // check parent
+      return getCallableMethod(clazz.getSuperclass());
+    } catch (SecurityException e) {
+      // This should never happen
+      throw new RuntimeException("Unexpected exception", e);
+    }
+  }
+
+  void verifyCaller(Callable callable) {
+    Method callMethod = getCallableMethod(callable.getClass());
+    for (Class exceptionOrError : callMethod.getExceptionTypes()) {
+      Preconditions.checkArgument(Exception.class.isAssignableFrom(exceptionOrError),
+          "Callable method exceptions must be dervied from Exception");
+      @SuppressWarnings("unchecked") Class exception =
+          (Class) exceptionOrError;
+      Preconditions.checkArgument(findMostSpecificRetryInfo(retryInfos, exception) != null,
+          "Declared exception '" + exception + "' is not covered by exception handler");
+    }
+  }
+
+  public ImmutableSet> getRetriableExceptions() {
+    return retriableExceptions;
+  }
+
+  public ImmutableSet> getNonRetriableExceptions() {
+    return nonRetriableExceptions;
+  }
+
+  boolean shouldRetry(Exception ex) {
+    RetryInfo retryInfo = findMostSpecificRetryInfo(retryInfos, ex.getClass());
+    return retryInfo == null ? false : retryInfo.retry;
+  }
+
+  /**
+   * Returns an instance which retry any checked exception and abort on any runtime exception.
+   */
+  public static ExceptionHandler getDefaultInstance() {
+    return DEFAULT_INSTANCE;
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+}
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
new file mode 100644
index 000000000000..ac4489684e3e
--- /dev/null
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -0,0 +1,228 @@
+package com.google.gcloud;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.Math.min;
+import static java.lang.Math.pow;
+import static java.lang.Math.random;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.base.Stopwatch;
+
+import java.io.InterruptedIOException;
+import java.nio.channels.ClosedByInterruptException;
+import java.util.concurrent.Callable;
+import java.util.logging.Logger;
+
+/**
+ * Utility class for retrying operations. For more details about the parameters, see
+ * {@link RetryParams}. If the request is never successful, a {@link RetriesExhaustedException} will
+ * be thrown.
+ *
+ * @param  return value of the closure that is being run with retries
+ */
+public class RetryHelper {
+
+  private static final Logger log = Logger.getLogger(RetryHelper.class.getName());
+
+  private final Stopwatch stopwatch;
+  private final Callable callable;
+  private final RetryParams params;
+  private final ExceptionHandler exceptionHandler;
+  private int attemptNumber;
+
+
+  private static final ThreadLocal context = new ThreadLocal<>();
+
+  public static class RetryHelperException extends RuntimeException {
+
+    private static final long serialVersionUID = -2907061015610448235L;
+
+    RetryHelperException() {}
+
+    RetryHelperException(String message) {
+      super(message);
+    }
+
+    RetryHelperException(Throwable cause) {
+      super(cause);
+    }
+
+    RetryHelperException(String message, Throwable cause) {
+      super(message, cause);
+    }
+  }
+
+  /**
+   * Thrown when a RetryHelper failed to complete its work due to interruption. Throwing this
+   * exception also sets the thread interrupt flag.
+   */
+  public static final class RetryInterruptedException extends RetryHelperException {
+
+    private static final long serialVersionUID = 1678966737697204885L;
+
+    RetryInterruptedException() {}
+
+    /**
+     * Sets the caller thread interrupt flag and throws {@code RetryInteruptedException}.
+     */
+    static void propagate() throws RetryInterruptedException {
+      Thread.currentThread().interrupt();
+      throw new RetryInterruptedException();
+    }
+  }
+
+  /**
+   * Thrown when a RetryHelper has attempted the maximum number of attempts allowed by RetryParams
+   * and was not successful.
+   */
+  public static final class RetriesExhaustedException extends RetryHelperException {
+
+    private static final long serialVersionUID = 780199686075408083L;
+
+    RetriesExhaustedException(String message) {
+      super(message);
+    }
+
+    RetriesExhaustedException(String message, Throwable cause) {
+      super(message, cause);
+    }
+  }
+
+  /**
+   * Thrown when RetryHelper callable has indicate it should not be retried.
+   */
+  public static final class NonRetriableException extends RetryHelperException {
+
+    private static final long serialVersionUID = -2331878521983499652L;
+
+    NonRetriableException(Throwable ex) {
+      super(ex);
+    }
+  }
+
+  static class Context {
+
+    private final RetryHelper helper;
+
+    Context(RetryHelper helper) {
+      this.helper = helper;
+    }
+
+    public RetryParams getRetryParams() {
+      return helper.params;
+    }
+
+    public int getAttemptNumber() {
+      return helper.attemptNumber;
+    }
+  }
+
+  @VisibleForTesting
+  static final void setContext(Context ctx) {
+    if (ctx == null) {
+      context.remove();
+    } else {
+      context.set(ctx);
+    }
+  }
+
+  static final Context getContext() {
+    return context.get();
+  }
+
+  @VisibleForTesting
+  RetryHelper(Callable callable, RetryParams params, ExceptionHandler exceptionHandler,
+      Stopwatch stopwatch) {
+    this.callable = checkNotNull(callable);
+    this.params = checkNotNull(params);
+    this.stopwatch = checkNotNull(stopwatch);
+    this.exceptionHandler = checkNotNull(exceptionHandler);
+    exceptionHandler.verifyCaller(callable);
+  }
+
+  @Override
+  public String toString() {
+    ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+    toStringHelper.add("stopwatch", stopwatch);
+    toStringHelper.add("attempNumber", attemptNumber);
+    toStringHelper.add("callable", callable);
+    return toStringHelper.toString();
+  }
+
+  private V doRetry() throws RetryHelperException {
+    stopwatch.start();
+    while (true) {
+      attemptNumber++;
+      Exception exception;
+      try {
+        V value = callable.call();
+        if (attemptNumber > 1) {
+          log.fine(this + ": attempt #" + attemptNumber + " succeeded");
+        }
+        return value;
+      } catch (Exception e) {
+        if (!exceptionHandler.shouldRetry(e)) {
+          if (e instanceof InterruptedException || e instanceof InterruptedIOException
+              || e instanceof ClosedByInterruptException) {
+            RetryInterruptedException.propagate();
+          }
+          throw new NonRetriableException(e);
+        }
+        exception = e;
+      }
+      if (attemptNumber >= params.getRetryMaxAttempts()
+          || attemptNumber >= params.getRetryMinAttempts()
+          && stopwatch.elapsed(MILLISECONDS) >= params.getTotalRetryPeriodMillis()) {
+        throw new RetriesExhaustedException(this + ": Too many failures, giving up", exception);
+      }
+      long sleepDurationMillis = getSleepDuration(params, attemptNumber);
+      log.fine(this + ": Attempt #" + attemptNumber + " failed [" + exception + "], sleeping for "
+          + sleepDurationMillis + " ms");
+      try {
+        Thread.sleep(sleepDurationMillis);
+      } catch (InterruptedException e) {
+        RetryInterruptedException.propagate();
+      }
+    }
+  }
+
+  @VisibleForTesting
+  static long getSleepDuration(RetryParams retryParams, int attemptsSoFar) {
+    long initialDelay = retryParams.getInitialRetryDelayMillis();
+    double backoffFactor = retryParams.getRetryDelayBackoffFactor();
+    long maxDelay = retryParams.getMaxRetryDelayMillis();
+    long retryDelay = getExponentialValue(initialDelay, backoffFactor, maxDelay, attemptsSoFar);
+    return (long) ((random() / 2.0 + .75) * retryDelay);
+  }
+
+  private static long getExponentialValue(long initialDelay, double backoffFactor, long maxDelay,
+      int attemptsSoFar) {
+    return (long) min(maxDelay, pow(backoffFactor, min(1, attemptsSoFar) - 1) * initialDelay);
+  }
+
+  public static  V runWithRetries(Callable callable) throws RetryHelperException {
+    return runWithRetries(callable, RetryParams.getDefaultInstance(),
+        ExceptionHandler.getDefaultInstance());
+  }
+
+  public static  V runWithRetries(Callable callable, RetryParams params,
+      ExceptionHandler exceptionHandler) throws RetryHelperException {
+    return runWithRetries(callable, params, exceptionHandler, Stopwatch.createUnstarted());
+  }
+
+  @VisibleForTesting
+  static  V runWithRetries(Callable callable, RetryParams params,
+      ExceptionHandler exceptionHandler, Stopwatch stopwatch) throws RetryHelperException {
+    RetryHelper retryHelper = new RetryHelper<>(callable, params, exceptionHandler, stopwatch);
+    Context previousContext = getContext();
+    setContext(new Context(retryHelper));
+    try {
+      return retryHelper.doRetry();
+    } finally {
+      setContext(previousContext);
+    }
+  }
+}
diff --git a/src/main/java/com/google/gcloud/RetryParams.java b/src/main/java/com/google/gcloud/RetryParams.java
new file mode 100644
index 000000000000..cd500cd4149f
--- /dev/null
+++ b/src/main/java/com/google/gcloud/RetryParams.java
@@ -0,0 +1,272 @@
+package com.google.gcloud;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Parameters for configuring retries with an exponential backoff.
+ * Initial request is executed immediately. If the request fails but passes the
+ * {@link ExceptionHandler} criteria the calling thread sleeps for {@code initialRetryDelayMillis}.
+ * Each subsequent failure the sleep interval is calculated as:
+ * 

+ * {@code retryDelayBackoffFactor ^ attempts * initialRetryDelayMillis} but would be upper-bounded
+ * to {@code maxRetryDelayMillis}
+ * 

+ * This proceeds until either the request is successful, {@code retryMaxAttempts} are made, or both
+ * {@code retryMinAttempts} are made and {@code totalRetryPeriodMillis} have elapsed. To construct
+ * {@code RetryParams}, first create a {@link RetryParams.Builder}. The builder is mutable and each
+ * of the parameters can be set (any unset parameters will fallback to the defaults). The
+ * {@code Builder} can be then used to create an immutable {@code RetryParams} object. For default
+ * {@code RetryParams} use {@link #getDefaultInstance}. Default settings are subject to change
+ * release to release. If you require specific settings, explicitly create an instance of
+ * {@code RetryParams} with all the required settings.
+ *
+ * @see RetryHelper
+ */
+public final class RetryParams implements Serializable {
+
+  private static final long serialVersionUID = -8492751576749007700L;
+
+  public static final int DEFAULT_RETRY_MIN_ATTEMPTS = 3;
+  public static final int DEFAULT_RETRY_MAX_ATTEMPTS = 6;
+  public static final long DEFAULT_INITIAL_RETRY_DELAY_MILLIS = 250L;
+  public static final long DEFAULT_MAX_RETRY_DELAY_MILLIS = 10_000L;
+  public static final double DEFAULT_RETRY_DELAY_BACKOFF_FACTOR = 2.0;
+  public static final long DEFAULT_TOTAL_RETRY_PERIOD_MILLIS = 50_000L;
+
+  private final int retryMinAttempts;
+  private final int retryMaxAttempts;
+  private final long initialRetryDelayMillis;
+  private final long maxRetryDelayMillis;
+  private final double retryDelayBackoffFactor;
+  private final long totalRetryPeriodMillis;
+
+  private static final RetryParams DEFAULT_INSTANCE = new RetryParams(new Builder());
+
+
+  /**
+   * Create a new RetryParams with the parameters from a {@link RetryParams.Builder}
+   *
+   * @param builder the parameters to use to construct the RetryParams object
+   */
+  private RetryParams(Builder builder) {
+    retryMinAttempts = builder.retryMinAttempts;
+    retryMaxAttempts = builder.retryMaxAttempts;
+    initialRetryDelayMillis = builder.initialRetryDelayMillis;
+    maxRetryDelayMillis = builder.maxRetryDelayMillis;
+    retryDelayBackoffFactor = builder.retryDelayBackoffFactor;
+    totalRetryPeriodMillis = builder.totalRetryPeriodMillis;
+    checkArgument(retryMinAttempts >= 0, "retryMinAttempts must not be negative");
+    checkArgument(retryMaxAttempts >= retryMinAttempts,
+        "retryMaxAttempts must not be smaller than retryMinAttempts");
+    checkArgument(initialRetryDelayMillis >= 0, "initialRetryDelayMillis must not be negative");
+    checkArgument(maxRetryDelayMillis >= initialRetryDelayMillis,
+        "maxRetryDelayMillis must not be smaller than initialRetryDelayMillis");
+    checkArgument(retryDelayBackoffFactor >= 0, "retryDelayBackoffFactor must not be negative");
+    checkArgument(totalRetryPeriodMillis >= 0, "totalRetryPeriodMillis must not be negative");
+  }
+
+  /**
+   * Returns an instance with the default parameters.
+   */
+  public static RetryParams getDefaultInstance() {
+    return DEFAULT_INSTANCE;
+  }
+
+  /**
+   * RetryParams builder.
+   */
+  public static final class Builder {
+
+    private int retryMinAttempts;
+    private int retryMaxAttempts;
+    private long initialRetryDelayMillis;
+    private long maxRetryDelayMillis;
+    private double retryDelayBackoffFactor;
+    private long totalRetryPeriodMillis;
+
+    private Builder() {
+      this(null);
+    }
+
+    Builder(/* Nullable */RetryParams retryParams) {
+      if (retryParams == null) {
+        retryMinAttempts = DEFAULT_RETRY_MIN_ATTEMPTS;
+        retryMaxAttempts = DEFAULT_RETRY_MAX_ATTEMPTS;
+        initialRetryDelayMillis = DEFAULT_INITIAL_RETRY_DELAY_MILLIS;
+        maxRetryDelayMillis = DEFAULT_MAX_RETRY_DELAY_MILLIS;
+        retryDelayBackoffFactor = DEFAULT_RETRY_DELAY_BACKOFF_FACTOR;
+        totalRetryPeriodMillis = DEFAULT_TOTAL_RETRY_PERIOD_MILLIS;
+      } else {
+        retryMinAttempts = retryParams.getRetryMinAttempts();
+        retryMaxAttempts = retryParams.getRetryMaxAttempts();
+        initialRetryDelayMillis = retryParams.getInitialRetryDelayMillis();
+        maxRetryDelayMillis = retryParams.getMaxRetryDelayMillis();
+        retryDelayBackoffFactor = retryParams.getRetryDelayBackoffFactor();
+        totalRetryPeriodMillis = retryParams.getTotalRetryPeriodMillis();
+      }
+    }
+
+    /**
+     * Sets retryMinAttempts.
+     *
+     * @param retryMinAttempts the retryMinAttempts to set
+     * @return the Builder for chaining
+     */
+    public Builder retryMinAttempts(int retryMinAttempts) {
+      this.retryMinAttempts = retryMinAttempts;
+      return this;
+    }
+
+    /**
+     * Sets retryMaxAttempts.
+     *
+     * @param retryMaxAttempts the retryMaxAttempts to set
+     * @return the Builder for chaining
+     */
+    public Builder retryMaxAttempts(int retryMaxAttempts) {
+      this.retryMaxAttempts = retryMaxAttempts;
+      return this;
+    }
+
+    /**
+     * Sets initialRetryDelayMillis.
+     *
+     * @param initialRetryDelayMillis the initialRetryDelayMillis to set
+     * @return the Builder for chaining
+     */
+    public Builder initialRetryDelayMillis(long initialRetryDelayMillis) {
+      this.initialRetryDelayMillis = initialRetryDelayMillis;
+      return this;
+    }
+
+    /**
+     * Sets maxRetryDelayMillis.
+     *
+     * @param maxRetryDelayMillis the maxRetryDelayMillis to set
+     * @return the Builder for chaining
+     */
+    public Builder maxRetryDelayMillis(long maxRetryDelayMillis) {
+      this.maxRetryDelayMillis = maxRetryDelayMillis;
+      return this;
+    }
+
+    /**
+     * Sets retryDelayBackoffFactor.
+     *
+     * @param retryDelayBackoffFactor the retryDelayBackoffFactor to set
+     * @return the Builder for chaining
+     */
+    public Builder retryDelayBackoffFactor(double retryDelayBackoffFactor) {
+      this.retryDelayBackoffFactor = retryDelayBackoffFactor;
+      return this;
+    }
+
+    /**
+     * Sets totalRetryPeriodMillis.
+     *
+     * @param totalRetryPeriodMillis the totalRetryPeriodMillis to set
+     * @return the Builder for chaining
+     */
+    public Builder totalRetryPeriodMillis(long totalRetryPeriodMillis) {
+      this.totalRetryPeriodMillis = totalRetryPeriodMillis;
+      return this;
+    }
+
+    /**
+     * Create an instance of RetryParams with the parameters set in this builder.
+     *
+     * @return a new instance of RetryParams
+     */
+    public RetryParams build() {
+      return new RetryParams(this);
+    }
+  }
+
+  /**
+   * Returns the retryMinAttempts.
+   */
+  public int getRetryMinAttempts() {
+    return retryMinAttempts;
+  }
+
+  /**
+   * Returns the retryMaxAttempts.
+   */
+  public int getRetryMaxAttempts() {
+    return retryMaxAttempts;
+  }
+
+  /**
+   * Returns the initialRetryDelayMillis.
+   */
+  public long getInitialRetryDelayMillis() {
+    return initialRetryDelayMillis;
+  }
+
+  /**
+   * Returns the maxRetryDelayMillis.
+   */
+  public long getMaxRetryDelayMillis() {
+    return maxRetryDelayMillis;
+  }
+
+  /**
+   * Returns the maxRetryDelayBackoffFactor.
+   */
+  public double getRetryDelayBackoffFactor() {
+    return retryDelayBackoffFactor;
+  }
+
+  /**
+   * Returns the totalRetryPeriodMillis.
+   */
+  public long getTotalRetryPeriodMillis() {
+    return totalRetryPeriodMillis;
+  }
+
+
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(retryMinAttempts, retryMaxAttempts, initialRetryDelayMillis,
+        maxRetryDelayMillis, retryDelayBackoffFactor, totalRetryPeriodMillis);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+    if (!(obj instanceof RetryParams)) {
+      return false;
+    }
+    RetryParams other = (RetryParams) obj;
+    return retryMinAttempts == other.retryMinAttempts && retryMaxAttempts == other.retryMaxAttempts
+        && initialRetryDelayMillis == other.initialRetryDelayMillis
+        && maxRetryDelayMillis == other.maxRetryDelayMillis
+        && retryDelayBackoffFactor == other.retryDelayBackoffFactor
+        && totalRetryPeriodMillis == other.totalRetryPeriodMillis;
+  }
+
+  @Override
+  public String toString() {
+    ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+    toStringHelper.add("retryMinAttempts", retryMinAttempts);
+    toStringHelper.add("retryMaxAttempts", retryMaxAttempts);
+    toStringHelper.add("initialRetryDelayMillis", initialRetryDelayMillis);
+    toStringHelper.add("maxRetryDelayMillis", maxRetryDelayMillis);
+    toStringHelper.add("retryDelayBackoffFactor", retryDelayBackoffFactor);
+    toStringHelper.add("totalRetryPeriodMillis", totalRetryPeriodMillis);
+    return toStringHelper.toString();
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+}
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 826f59e1d300..24510adf7688 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -116,4 +116,6 @@ public AuthConfig authConfig() {
   protected HttpRequestInitializer httpRequestInitializer() {
     return authConfig().httpRequestInitializer(httpTransport, scopes());
   }
+
+  public abstract Builder toBuilder();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index c0d5e3aee3a1..4931bd62af71 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -95,6 +95,11 @@ protected Set scopes() {
     return SCOPES;
   }
 
+  @Override
+  public Builder toBuilder() {
+    return new Builder(this);
+  }
+
   DatastoreOptions toDatastoreOptions() {
     return new DatastoreOptions.Builder()
         .dataset(dataset())
@@ -106,8 +111,4 @@ DatastoreOptions toDatastoreOptions() {
   public static Builder builder() {
     return new Builder();
   }
-
-  public static Builder builder(DatastoreServiceOptions options) {
-    return new Builder(options);
-  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index b186edb83b0e..7862681038c5 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -1,6 +1,7 @@
 /**
  * A client to the Google Cloud Datastore.
- * A simple usage example:
+ *
+ * 
A simple usage example:
  * 
 {@code
  * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
@@ -29,5 +30,7 @@
  *   }
  * }
  * } 

+ *
+ * @see Google Cloud Datastore
  */
 package com.google.gcloud.datastore;
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
new file mode 100644
index 000000000000..06f2c467bbaf
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -0,0 +1,5 @@
+package com.google.gcloud.storage;
+
+public interface Acl {
+
+}
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
new file mode 100644
index 000000000000..fa7bae1da7fd
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -0,0 +1,35 @@
+package com.google.gcloud.storage;
+
+import java.nio.ByteBuffer;
+
+public interface Bucket {
+
+  String name();
+
+  Acl acl();
+
+  void updateAcl(Acl acl);
+
+  Acl defaultObjectAcl();
+
+  void updateDefaultObjectAcl();
+
+  Acl acl(String objectName);
+
+  void updateAcl(String objectName, Acl acl);
+
+  void delete(String... objectName);
+
+  void compose(Iterable source, String dest);
+
+  void copy(String source, String dest);
+
+  // TODO (ozarov): consider replace with Object that has a reference to bucket and name
+  // that object can return its own meta-data, update its own meta-data, replace its content
+  // via a stream or byteBuffer, read its content (via stream or ByteBuffer),...
+  //void copy(String source, String bucket, String dest);
+
+  void put(String name, ByteBuffer bytes);
+
+  // TODO: add listing
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
new file mode 100644
index 000000000000..52e2c879c0f8
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -0,0 +1,9 @@
+package com.google.gcloud.storage;
+
+public interface StorageService {
+
+  Iterable listBuckets();
+
+  Bucket getBucket(String bucket);
+
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
new file mode 100644
index 000000000000..acc243b9ba38
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
@@ -0,0 +1,19 @@
+package com.google.gcloud.storage;
+
+
+
+public abstract class StorageServiceFactory {
+
+  private static final StorageServiceFactory INSTANCE = new StorageServiceFactory() {
+      @Override
+      public StorageService get(StorageServiceOptions options) {
+        return new StorageServiceImpl(options);
+      }
+    };
+
+  public static StorageService getDefault(StorageServiceOptions options) {
+    return INSTANCE.get(options);
+  }
+
+  public abstract StorageService get(StorageServiceOptions options);
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
new file mode 100644
index 000000000000..135bff666c0f
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -0,0 +1,28 @@
+package com.google.gcloud.storage;
+
+import com.google.api.services.storage.Storage;
+
+final class StorageServiceImpl implements StorageService {
+
+  private final StorageServiceOptions options;
+  private final Storage storage;
+
+  StorageServiceImpl(StorageServiceOptions options) {
+    this.options = options;
+    this.storage = options.getStorage();
+  }
+
+  @Override
+  public Iterable listBuckets() {
+    // TODO Auto-generated method stub
+    return null;
+  }
+
+  @Override
+  public Bucket getBucket(String bucket) {
+    // TODO Auto-generated method stub
+    return null;
+  }
+
+
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
new file mode 100644
index 000000000000..3fa48eb5095c
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -0,0 +1,100 @@
+package com.google.gcloud.storage;
+
+import com.google.api.client.json.jackson.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.gcloud.ServiceOptions;
+
+import java.lang.reflect.Method;
+import java.util.Set;
+
+public class StorageServiceOptions extends ServiceOptions {
+
+  private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control";
+  private static final Set SCOPES = ImmutableSet.of(GCS_SCOPE);
+  private static final String DEFAULT_PATH_DELIMITER = "/";
+
+  private final String project;
+  private final String pathDelimiter;
+
+  StorageServiceOptions(Builder builder) {
+    super(builder);
+    pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
+    project = builder.project != null ? builder.project :  getAppEngineProject();
+    Preconditions.checkArgument(project != null, "Missing required project id");
+  }
+
+  private static String getAppEngineProject() {
+    // TODO(ozarov): An alternative to reflection would be to depend on AE api jar:
+    // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0
+    try {
+      Class factoryClass =
+          Class.forName("com.google.appengine.api.appidentity.AppIdentityServiceFactory");
+      Method method = factoryClass.getMethod("getAppIdentityService");
+      Object appIdentityService = method.invoke(null);
+      method = appIdentityService.getClass().getMethod("getServiceAccountName");
+      String serviceAccountName = (String) method.invoke(appIdentityService);
+      int indexOfAtSign = serviceAccountName.indexOf('@');
+      return serviceAccountName.substring(0, indexOfAtSign);
+    } catch (Exception ex) {
+      return null;
+    }
+  }
+
+  public static class Builder extends ServiceOptions.Builder {
+
+    private String project;
+    private String pathDelimiter;
+
+    private Builder() {
+    }
+
+    private Builder(StorageServiceOptions options) {
+      super(options);
+    }
+
+    public Builder project(String project) {
+      this.project = project;
+      return this;
+    }
+
+    public Builder pathDelimiter(String pathDelimiter) {
+      this.pathDelimiter = pathDelimiter;
+      return this;
+    }
+
+    @Override
+    public StorageServiceOptions build() {
+      return new StorageServiceOptions(this);
+    }
+  }
+
+  @Override
+  protected Set scopes() {
+    return SCOPES;
+  }
+
+  Storage getStorage() {
+    return new Storage.Builder(httpTransport(), new JacksonFactory(), httpRequestInitializer())
+        .build();
+  }
+
+  public String pathDelimiter() {
+    return pathDelimiter;
+  }
+
+  @Override
+  public Builder toBuilder() {
+    return new Builder(this);
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  public static Builder builder(StorageServiceOptions options) {
+    return new Builder(options);
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/src/main/java/com/google/gcloud/storage/package-info.java
new file mode 100644
index 000000000000..0f78d1bab46a
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * A client to Google Cloud Storage.
+ *
+ * @see Google Cloud Storageg
+ */
+package com.google.gcloud.storage;
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
new file mode 100644
index 000000000000..f88c5211ad93
--- /dev/null
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -0,0 +1,113 @@
+package com.google.gcloud;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.channels.ClosedByInterruptException;
+import java.util.concurrent.Callable;
+
+/**
+ * Tests for {@link ExceptionHandler}.
+ */
+@RunWith(JUnit4.class)
+public class ExceptionHandlerTest {
+
+  @Test
+  public void testVerifyCaller() {
+    class A implements Callable {
+      @Override
+      public Object call() throws IOException, InterruptedException {
+        return null;
+      }
+    }
+
+    class B extends A {
+    }
+
+    class C extends A {
+      @Override
+      public Object call() throws FileNotFoundException {
+        return null;
+      }
+    }
+
+    class D extends C {
+      @Override
+      public Object call() throws IllegalArgumentException {
+        return null;
+      }
+    }
+
+    class E extends A {
+      @Override
+      public String call() throws NullPointerException {
+        return null;
+      }
+    }
+
+    class F extends A {
+      @Override
+      public Object call() throws Error {
+        return null;
+      }
+    }
+
+    // using default exception handler (retry upon any non-runtime exceptions)
+    ExceptionHandler handler = ExceptionHandler.getDefaultInstance();
+    assertValidCallable(new A(), handler);
+    assertValidCallable(new B(), handler);
+    assertValidCallable(new C(), handler);
+    assertValidCallable(new D(), handler);
+    assertValidCallable(new E(), handler);
+    assertInvalidCallable(new F(), handler);
+
+    handler = ExceptionHandler.builder()
+        .retryOn(FileNotFoundException.class, NullPointerException.class)
+        .build();
+    assertInvalidCallable(new A(), handler);
+    assertInvalidCallable(new B(), handler);
+    assertValidCallable(new C(), handler);
+    assertInvalidCallable(new D(), handler);
+    assertValidCallable(new E(), handler);
+    assertInvalidCallable(new F(), handler);
+  }
+
+  private static  void assertValidCallable(Callable callable, ExceptionHandler handler) {
+    handler.verifyCaller(callable);
+  }
+
+  private static  void assertInvalidCallable(Callable callable, ExceptionHandler handler) {
+    try {
+      handler.verifyCaller(callable);
+      fail("Expected RetryHelper constructor to fail");
+    } catch (IllegalArgumentException ex) {
+      // expected
+    }
+  }
+
+  @Test
+  public void testShouldTry() {
+    ExceptionHandler handler = ExceptionHandler.builder().retryOn(IOException.class).build();
+    assertTrue(handler.shouldRetry(new IOException()));
+    assertTrue(handler.shouldRetry(new ClosedByInterruptException()));
+    assertFalse(handler.shouldRetry(new RuntimeException()));
+
+    handler = ExceptionHandler.builder()
+        .retryOn(IOException.class, NullPointerException.class)
+        .abortOn(RuntimeException.class, ClosedByInterruptException.class,
+            InterruptedException.class)
+        .build();
+    assertTrue(handler.shouldRetry(new IOException()));
+    assertFalse(handler.shouldRetry(new ClosedByInterruptException()));
+    assertFalse(handler.shouldRetry(new InterruptedException()));
+    assertFalse(handler.shouldRetry(new RuntimeException()));
+    assertTrue(handler.shouldRetry(new NullPointerException()));
+  }
+}
diff --git a/src/test/java/com/google/gcloud/RetryParamsTest.java b/src/test/java/com/google/gcloud/RetryParamsTest.java
new file mode 100644
index 000000000000..b59324f0cb1e
--- /dev/null
+++ b/src/test/java/com/google/gcloud/RetryParamsTest.java
@@ -0,0 +1,91 @@
+package com.google.gcloud;
+
+import static com.google.gcloud.RetryParams.DEFAULT_INITIAL_RETRY_DELAY_MILLIS;
+import static com.google.gcloud.RetryParams.DEFAULT_MAX_RETRY_DELAY_MILLIS;
+import static com.google.gcloud.RetryParams.DEFAULT_RETRY_DELAY_BACKOFF_FACTOR;
+import static com.google.gcloud.RetryParams.DEFAULT_RETRY_MAX_ATTEMPTS;
+import static com.google.gcloud.RetryParams.DEFAULT_RETRY_MIN_ATTEMPTS;
+import static com.google.gcloud.RetryParams.DEFAULT_TOTAL_RETRY_PERIOD_MILLIS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.gcloud.RetryParams.Builder;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.Arrays;
+
+/**
+ * Tests for {@link RetryParams}.
+ */
+@RunWith(JUnit4.class)
+public class RetryParamsTest {
+
+  @Test
+  public void testDefaults() {
+    RetryParams params1 = RetryParams.getDefaultInstance();
+    RetryParams params2 = RetryParams.builder().build();
+    for (RetryParams params : Arrays.asList(params1, params2)) {
+      assertEquals(DEFAULT_INITIAL_RETRY_DELAY_MILLIS, params.getInitialRetryDelayMillis());
+      assertEquals(DEFAULT_MAX_RETRY_DELAY_MILLIS, params.getMaxRetryDelayMillis());
+      assertEquals(DEFAULT_RETRY_DELAY_BACKOFF_FACTOR, params.getRetryDelayBackoffFactor(), 0);
+      assertEquals(DEFAULT_RETRY_MAX_ATTEMPTS, params.getRetryMaxAttempts());
+      assertEquals(DEFAULT_RETRY_MIN_ATTEMPTS, params.getRetryMinAttempts());
+      assertEquals(DEFAULT_TOTAL_RETRY_PERIOD_MILLIS, params.getTotalRetryPeriodMillis());
+    }
+  }
+
+  @Test
+  public void testSetAndCopy() {
+    RetryParams.Builder builder = RetryParams.builder();
+    builder.initialRetryDelayMillis(101);
+    builder.maxRetryDelayMillis(102);
+    builder.retryDelayBackoffFactor(103);
+    builder.retryMinAttempts(107);
+    builder.retryMaxAttempts(108);
+    builder.totalRetryPeriodMillis(109);
+    RetryParams params1 = builder.build();
+    RetryParams params2 = new RetryParams.Builder(params1).build();
+    for (RetryParams params : Arrays.asList(params1, params2)) {
+      assertEquals(101, params.getInitialRetryDelayMillis());
+      assertEquals(102, params.getMaxRetryDelayMillis());
+      assertEquals(103, params.getRetryDelayBackoffFactor(), 0);
+      assertEquals(107, params.getRetryMinAttempts());
+      assertEquals(108, params.getRetryMaxAttempts());
+      assertEquals(109, params.getTotalRetryPeriodMillis());
+    }
+  }
+
+  @Test
+  public void testBadSettings() {
+    RetryParams.Builder builder = RetryParams.builder();
+    builder.initialRetryDelayMillis(-1);
+    builder = verifyFailure(builder);
+    builder.maxRetryDelayMillis(RetryParams.getDefaultInstance().getInitialRetryDelayMillis() - 1);
+    builder = verifyFailure(builder);
+    builder.retryDelayBackoffFactor(-1);
+    builder = verifyFailure(builder);
+    builder.retryMinAttempts(-1);
+    builder = verifyFailure(builder);
+    builder.retryMaxAttempts(RetryParams.getDefaultInstance().getRetryMinAttempts() - 1);
+    builder = verifyFailure(builder);
+    builder.totalRetryPeriodMillis(-1);
+    builder = verifyFailure(builder);
+    // verify that it is OK for min and max to be equal
+    builder.retryMaxAttempts(RetryParams.getDefaultInstance().getRetryMinAttempts());
+    builder.maxRetryDelayMillis(RetryParams.getDefaultInstance().getInitialRetryDelayMillis());
+    builder.build();
+  }
+
+  private static Builder verifyFailure(Builder builder) {
+    try {
+      builder.build();
+      fail("Expected IllegalArgumentException");
+    } catch (IllegalArgumentException ex) {
+      // expected
+    }
+    return RetryParams.builder();
+  }
+}

From 73b11399ff3401f6c716d1c82feaa4a45d85f23b Mon Sep 17 00:00:00 2001
From: Patrick Costello 
Date: Wed, 17 Dec 2014 14:33:44 -0800
Subject: [PATCH 060/732] Rename comit to commit

Fix a typo in a private method.
---
 .../gcloud/datastore/DatastoreServiceImpl.java     | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index d55aaa47a380..b41fed8b0ee5 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -161,7 +161,7 @@ public void add(Entity... entities) {
       }
       mutationPb.addInsert(entity.toPb());
     }
-    comitMutation(mutationPb);
+    commitMutation(mutationPb);
   }
 
   @Override
@@ -174,7 +174,7 @@ public void update(Entity... entities) {
     for (Entity entity : dedupEntities.values()) {
       mutationPb.addUpdate(entity.toPb());
     }
-    comitMutation(mutationPb);
+    commitMutation(mutationPb);
   }
 
   @Override
@@ -187,7 +187,7 @@ public void put(Entity... entities) {
     for (Entity e : dedupEntities.values()) {
       mutationPb.addUpsert(e.toPb());
     }
-    comitMutation(mutationPb);
+    commitMutation(mutationPb);
   }
 
   @Override
@@ -197,20 +197,20 @@ public void delete(Key... keys) {
     for (Key key : dedupKeys) {
       mutationPb.addDelete(key.toPb());
     }
-    comitMutation(mutationPb);
+    commitMutation(mutationPb);
   }
 
-  private void comitMutation(DatastoreV1.Mutation.Builder mutationPb) {
+  private void commitMutation(DatastoreV1.Mutation.Builder mutationPb) {
     if (options.force()) {
       mutationPb.setForce(true);
     }
     DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
     requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL);
     requestPb.setMutation(mutationPb);
-    comitMutation(requestPb);
+    commitMutation(requestPb);
   }
 
-  void comitMutation(DatastoreV1.CommitRequest.Builder requestPb) {
+  void commitMutation(DatastoreV1.CommitRequest.Builder requestPb) {
     try {
       datastore.commit(requestPb.build());
     } catch (DatastoreException e) {

From 77d2970ba26af8c3b71c8f52c63ae6a558db605d Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 17 Dec 2014 15:46:03 -0800
Subject: [PATCH 061/732] Added DatastoreReaderWriter Renamed KeyBuilder
 utility to KeyFactory Add interceptors to exception handler

---
 .gitignore                                    |   1 +
 pom.xml                                       | 198 ------------------
 .../com/google/gcloud/ExceptionHandler.java   |  66 +++++-
 .../google/gcloud/datastore/BaseEntity.java   |  10 +-
 .../com/google/gcloud/datastore/BaseKey.java  |  17 +-
 .../gcloud/datastore/BatchWriterImpl.java     |   2 +-
 .../datastore/DatastoreReaderWriter.java      |   8 +
 .../gcloud/datastore/DatastoreService.java    |   8 +-
 .../datastore/DatastoreServiceImpl.java       |   5 -
 .../com/google/gcloud/datastore/Entity.java   |   6 +-
 .../java/com/google/gcloud/datastore/Key.java |   9 +-
 .../google/gcloud/datastore/KeyBuilder.java   |  45 ----
 .../google/gcloud/datastore/KeyFactory.java   |  45 ++++
 .../gcloud/datastore/PartialEntity.java       |   6 +-
 .../google/gcloud/datastore/PartialKey.java   |   7 +-
 .../google/gcloud/datastore/Transaction.java  |   4 +-
 .../google/gcloud/datastore/package-info.java |   2 +-
 .../com/google/gcloud/storage/Bucket.java     |   1 +
 .../google/gcloud/ExceptionHandlerTest.java   |  51 ++++-
 .../datastore/DatastoreServiceTest.java       |  28 +--
 20 files changed, 201 insertions(+), 318 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
 delete mode 100644 src/main/java/com/google/gcloud/datastore/KeyBuilder.java
 create mode 100644 src/main/java/com/google/gcloud/datastore/KeyFactory.java

diff --git a/.gitignore b/.gitignore
index b83d22266ac8..3ae51aeeae93 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 /target/
+.settings
diff --git a/pom.xml b/pom.xml
index 585f21738fa5..1fa28d56c920 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,214 +90,16 @@
     
   
   
-    /usr/local/google/home/ozarov/git/git-demo/src/main/java
-    /usr/local/google/home/ozarov/git/git-demo/src/main/scripts
-    /usr/local/google/home/ozarov/git/git-demo/src/test/java
-    /usr/local/google/home/ozarov/git/git-demo/target/classes
-    /usr/local/google/home/ozarov/git/git-demo/target/test-classes
-    
-      
-        /usr/local/google/home/ozarov/git/git-demo/src/main/resources
-      
-    
-    
-      
-        /usr/local/google/home/ozarov/git/git-demo/src/test/resources
-      
-    
-    /usr/local/google/home/ozarov/git/git-demo/target
-    git-demo-0.0.1-SNAPSHOT
-    
-      
-        
-          maven-antrun-plugin
-          1.3
-        
-        
-          maven-assembly-plugin
-          2.2-beta-5
-        
-        
-          maven-dependency-plugin
-          2.8
-        
-        
-          maven-release-plugin
-          2.3.2
-        
-      
-    
     
       
         maven-compiler-plugin
         3.1
-        
-          
-            default-testCompile
-            test-compile
-            
-              testCompile
-            
-            
-              1.7
-              1.7
-              UTF-8
-            
-          
-          
-            default-compile
-            compile
-            
-              compile
-            
-            
-              1.7
-              1.7
-              UTF-8
-            
-          
-        
         
           1.7
           1.7
           UTF-8
         
       
-      
-        maven-clean-plugin
-        2.5
-        
-          
-            default-clean
-            clean
-            
-              clean
-            
-          
-        
-      
-      
-        maven-install-plugin
-        2.4
-        
-          
-            default-install
-            install
-            
-              install
-            
-          
-        
-      
-      
-        maven-resources-plugin
-        2.6
-        
-          
-            default-resources
-            process-resources
-            
-              resources
-            
-          
-          
-            default-testResources
-            process-test-resources
-            
-              testResources
-            
-          
-        
-      
-      
-        maven-surefire-plugin
-        2.12.4
-        
-          
-            default-test
-            test
-            
-              test
-            
-          
-        
-      
-      
-        maven-jar-plugin
-        2.4
-        
-          
-            default-jar
-            package
-            
-              jar
-            
-          
-        
-      
-      
-        maven-deploy-plugin
-        2.7
-        
-          
-            default-deploy
-            deploy
-            
-              deploy
-            
-          
-        
-      
-      
-        maven-site-plugin
-        3.3
-        
-          
-            default-site
-            site
-            
-              site
-            
-            
-              /usr/local/google/home/ozarov/git/git-demo/target/site
-              
-                
-                  org.apache.maven.plugins
-                  maven-project-info-reports-plugin
-                
-              
-            
-          
-          
-            default-deploy
-            site-deploy
-            
-              deploy
-            
-            
-              /usr/local/google/home/ozarov/git/git-demo/target/site
-              
-                
-                  org.apache.maven.plugins
-                  maven-project-info-reports-plugin
-                
-              
-            
-          
-        
-        
-          /usr/local/google/home/ozarov/git/git-demo/target/site
-          
-            
-              org.apache.maven.plugins
-              maven-project-info-reports-plugin
-            
-          
-        
-      
     
   
-  
-    /usr/local/google/home/ozarov/git/git-demo/target/site
-  
 
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index 7a7371e175ab..9ad46f9d1040 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -4,6 +4,7 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
@@ -22,25 +23,65 @@ public final class ExceptionHandler implements Serializable {
   private static final ExceptionHandler DEFAULT_INSTANCE =
       builder().retryOn(Exception.class).abortOn(RuntimeException.class).build();
 
+  private final ImmutableList interceptors;
   private final ImmutableSet> retriableExceptions;
   private final ImmutableSet> nonRetriableExceptions;
   private final Set retryInfos = Sets.newHashSet();
 
+  public interface Interceptor extends Serializable {
+
+    /**
+     * This method is called before evaluating if the exception should be propagated
+     * and could short-circuit the evaluation process.
+     *
+     * @param exception the exception that is being evaluated
+     * @return {@code Boolean.TRUE} if exception should be ignored, {@code Boolean.FALSE}
+     *      if exception should be propagated or {@code null} if evaluation should proceed.
+     */
+    Boolean shouldRetry(Exception exception);
+
+    /**
+     * This method is called after the evaluation but could alter the result if desired.
+     *
+     * @param exception the exception that is being evaluated
+     * @param shouldRetry the result of the evaluation
+     * @return {@code true} if exception should be ignored or {@code false}
+     *      if exception should be propagated.
+     */
+    boolean shouldRetry(Exception exception, boolean shouldRetry);
+  }
+
   /**
    * ExceptionHandler builder.
    */
   public static class Builder {
 
+    private final ImmutableList.Builder interceptors = ImmutableList.builder();
     private final ImmutableSet.Builder> retriableExceptions =
-        new ImmutableSet.Builder<>();
+        ImmutableSet.builder();
     private final ImmutableSet.Builder> nonRetriableExceptions =
-        new ImmutableSet.Builder<>();
+        ImmutableSet.builder();
 
     private Builder() {
     }
 
+
+    /**
+     * Adds the exception handler interceptors.
+     * Call order will be maintained.
+
+     * @param interceptors the interceptors for this exception handler
+     * @return the Builder for chaining
+     */
+    public Builder interceptor(Interceptor... interceptors) {
+      for (Interceptor interceptor : interceptors) {
+        this.interceptors.add(interceptor);
+      }
+      return this;
+    }
+
     /**
-     * Specify on what exceptions to continue.
+     * Add the exceptions to ignore/retry-on.
      *
      * @param exceptions retry should continue when such exceptions are thrown
      * @return the Builder for chaining
@@ -54,7 +95,7 @@ public final Builder retryOn(Class... exceptions) {
     }
 
     /**
-     * Specify on what exceptions to abort.
+     * Adds the exceptions to abort on.
      *
      * @param exceptions retry should abort when such exceptions are thrown
      * @return the Builder for chaining
@@ -107,6 +148,7 @@ public boolean equals(Object obj) {
   }
 
   private ExceptionHandler(Builder builder) {
+    interceptors = builder.interceptors.build();
     retriableExceptions = builder.retriableExceptions.build();
     nonRetriableExceptions = builder.nonRetriableExceptions.build();
     Preconditions.checkArgument(
@@ -171,17 +213,27 @@ void verifyCaller(Callable callable) {
     }
   }
 
-  public ImmutableSet> getRetriableExceptions() {
+  public Set> getRetriableExceptions() {
     return retriableExceptions;
   }
 
-  public ImmutableSet> getNonRetriableExceptions() {
+  public Set> getNonRetriableExceptions() {
     return nonRetriableExceptions;
   }
 
   boolean shouldRetry(Exception ex) {
+    for (Interceptor interceptor : interceptors) {
+      Boolean shouldRetry = interceptor.shouldRetry(ex);
+      if (shouldRetry != null) {
+        return shouldRetry.booleanValue();
+      }
+    }
     RetryInfo retryInfo = findMostSpecificRetryInfo(retryInfos, ex.getClass());
-    return retryInfo == null ? false : retryInfo.retry;
+    boolean shouldRetry = retryInfo == null ? false : retryInfo.retry;
+    for (Interceptor interceptor : interceptors) {
+      shouldRetry = interceptor.shouldRetry(ex, shouldRetry);
+    }
+    return shouldRetry;
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 6d5784839e94..d5497bb0b308 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -30,9 +30,9 @@ abstract class BaseEntity extends Serializable {
 
   private final transient ImmutableSortedMap> properties;
 
-  protected abstract static class Builder> {
+  protected abstract static class Builder> {
 
-    private final Map> properties;
+    protected final Map> properties;
 
     protected Builder() {
       properties = new HashMap<>();
@@ -123,11 +123,7 @@ public B setNull(String name) {
       return self();
     }
 
-    public E build() {
-      return build(ImmutableSortedMap.copyOf(properties));
-    }
-
-    protected abstract E build(ImmutableSortedMap> properties);
+    public abstract BaseEntity build();
   }
 
   protected BaseEntity(ImmutableSortedMap> properties) {
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index 2812574f6dbe..d8a7a553c157 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -26,12 +26,12 @@ abstract class BaseKey extends Serializable {
   private final transient ImmutableList ancestors;
   private final transient String kind;
 
-  abstract static class Builder> {
+  abstract static class Builder> {
 
-    private String dataset;
-    private String namespace;
-    private String kind;
-    private final List ancestors;
+    protected String dataset;
+    protected String namespace;
+    protected String kind;
+    protected final List ancestors;
 
     private static final int MAX_PATH = 100;
 
@@ -100,12 +100,7 @@ public B namespace(String namespace) {
       return self();
     }
 
-    public K build() {
-      return build(dataset, namespace, ImmutableList.copyOf(ancestors), kind);
-    }
-
-    protected abstract K build(
-        String dataset, String namespace, ImmutableList ancestors, String kind);
+    protected abstract BaseKey build();
   }
 
   BaseKey(String dataset, String namespace, ImmutableList ancestors, String kind) {
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
index 2395c2968c54..9f760faee971 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
@@ -119,7 +119,7 @@ public void submit() {
     }
     DatastoreV1.CommitRequest.Builder requestPb = newCommitRequest();
     requestPb.setMutation(mutationPb);
-    datastore.comitMutation(requestPb);
+    datastore.commitMutation(requestPb);
     active = false;
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
new file mode 100644
index 000000000000..52304f4e7e27
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
@@ -0,0 +1,8 @@
+package com.google.gcloud.datastore;
+
+
+/**
+ * An interface that combines both Google Cloud Datastore read and write operations.
+ */
+public interface DatastoreReaderWriter extends DatastoreReader, DatastoreWriter {
+}
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index efd41bd9544e..45076277f701 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -5,19 +5,13 @@
 /**
  * An interface for Google Cloud Datastore dataset.
  */
-public interface DatastoreService extends DatastoreReader, DatastoreWriter {
+public interface DatastoreService extends DatastoreReaderWriter {
 
   /**
    * Returns the {@code DatastoreServiceOptions} for this service.
    */
   DatastoreServiceOptions options();
 
-  /**
-   * Returns a key builder for the requested {@code kind}.
-   * The key would be initialized with the this service dataset and default namespace.
-   */
-  KeyBuilder newKeyBuilder(String kind);
-
   /**
    * Returns a new Datastore transaction.
    *
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index b41fed8b0ee5..7af34773c3b8 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -32,11 +32,6 @@ public DatastoreServiceOptions options() {
     return options;
   }
 
-  @Override
-  public KeyBuilder newKeyBuilder(String kind) {
-    return new KeyBuilder(this, kind);
-  }
-
   @Override
   public BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption) {
     return new BatchWriterImpl(this, batchWriteOption);
diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java
index 5262fcccd8c1..2d4ae4306942 100644
--- a/src/main/java/com/google/gcloud/datastore/Entity.java
+++ b/src/main/java/com/google/gcloud/datastore/Entity.java
@@ -20,7 +20,7 @@ public final class Entity extends PartialEntity {
 
   private static final long serialVersionUID = 432961565733066915L;
 
-  public static final class Builder extends BaseEntity.Builder {
+  public static final class Builder extends BaseEntity.Builder {
 
     private Key key;
 
@@ -39,8 +39,8 @@ public Builder key(Key key) {
     }
 
     @Override
-    protected Entity build(ImmutableSortedMap> properties) {
-      return new Entity(key, properties);
+    public Entity build() {
+      return new Entity(key, ImmutableSortedMap.copyOf(properties));
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index 272fc6e4d7f4..a9b7ae6a0965 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -25,7 +25,7 @@ public final class Key extends PartialKey {
 
   private final transient PathElement leaf;
 
-  public static final class Builder extends BaseKey.Builder {
+  public static final class Builder extends BaseKey.Builder {
 
     private String name;
     private Long id;
@@ -62,12 +62,11 @@ public Builder id(long id) {
     }
 
     @Override
-    protected Key build(String dataset, String namespace, ImmutableList ancestors,
-        String kind) {
+    public Key build() {
       if (id == null) {
-        return new Key(dataset, namespace, ancestors, kind, name);
+        return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, name);
       }
-      return new Key(dataset, namespace, ancestors, kind, id);
+      return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, id);
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java b/src/main/java/com/google/gcloud/datastore/KeyBuilder.java
deleted file mode 100644
index a906a29d423c..000000000000
--- a/src/main/java/com/google/gcloud/datastore/KeyBuilder.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package com.google.gcloud.datastore;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import com.google.common.collect.ImmutableList;
-
-/**
- * An helper for creating keys for a specific {@link DatastoreService},
- * using its associated dataset and namespace.
- */
-public final class KeyBuilder extends BaseKey.Builder {
-
-  private final DatastoreService service;
-
-  /**
-   * Constructing a KeyBuilder.
-   */
-  public KeyBuilder(DatastoreService service, String kind) {
-    super(checkNotNull(service).options().dataset(), kind);
-    this.service = service;
-    namespace(service.options().namespace());
-  }
-
-  @Override
-  protected PartialKey build(String dataset, String namespace,
-      ImmutableList ancestors, String kind) {
-    return new PartialKey(dataset, namespace, ancestors, kind);
-  }
-
-  public Key build(String name) {
-    return build().newKey(name);
-  }
-
-  public Key build(long id) {
-    return build().newKey(id);
-  }
-
-  /**
-   * Builds a key with a newly allocated id.
-   * @throws DatastoreServiceException if allocation failed.
-   */
-  public Key allocateIdAndBuild() {
-    return service.allocateId(build());
-  }
-}
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
new file mode 100644
index 000000000000..f671e0f72da5
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -0,0 +1,45 @@
+package com.google.gcloud.datastore;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * An helper for creating keys for a specific {@link DatastoreService},
+ * using its associated dataset and namespace.
+ */
+public final class KeyFactory extends BaseKey.Builder {
+
+  private final DatastoreService service;
+
+  public KeyFactory(DatastoreService service, String kind) {
+    super(checkNotNull(service).options().dataset(), kind);
+    this.service = service;
+    namespace(service.options().namespace());
+  }
+
+  @Override
+  protected PartialKey build() {
+    return new PartialKey(dataset, namespace, ImmutableList.copyOf(ancestors), kind);
+  }
+
+  public PartialKey newKey() {
+    return build();
+  }
+
+  public Key newKey(String name) {
+    return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, name);
+  }
+
+  public Key newKey(long id) {
+    return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, id);
+  }
+
+  /**
+   * Return a key with a newly allocated id.
+   * @throws DatastoreServiceException if allocation failed.
+   */
+  public Key allocateId() {
+    return service.allocateId(build());
+  }
+}
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index b1c3dac6e970..9edefac1ca34 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -19,7 +19,7 @@ public class PartialEntity extends BaseEntity {
 
   private final transient PartialKey key;
 
-  public static class Builder extends BaseEntity.Builder {
+  public static class Builder extends BaseEntity.Builder {
 
     private PartialKey key;
 
@@ -37,8 +37,8 @@ public Builder key(PartialKey key) {
     }
 
     @Override
-    protected PartialEntity build(ImmutableSortedMap> properties) {
-      return new PartialEntity(key, properties);
+    public PartialEntity build() {
+      return new PartialEntity(key, ImmutableSortedMap.copyOf(properties));
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java
index f7e6c00374c8..7630f3ebf275 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java
@@ -15,7 +15,7 @@ public class PartialKey extends BaseKey {
 
   private static final long serialVersionUID = -75301206578793347L;
 
-  public static class Builder extends BaseKey.Builder {
+  public static class Builder extends BaseKey.Builder {
 
     private Builder(String dataset, String kind) {
       super(dataset, kind);
@@ -26,9 +26,8 @@ private Builder(PartialKey copyFrom) {
     }
 
     @Override
-    protected PartialKey build(String dataset, String namespace,
-        ImmutableList ancestors, String kind) {
-      return new PartialKey(dataset, namespace, ancestors, kind);
+    public PartialKey build() {
+      return new PartialKey(dataset, namespace, ImmutableList.copyOf(ancestors), kind);
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index d47a63ddbf1e..2a9267cf6d41 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -33,7 +33,7 @@
  * @see Google Cloud Datastore transactions
  *
  */
-public interface Transaction extends DatastoreReader, DatastoreWriter {
+public interface Transaction extends DatastoreReaderWriter {
 
   /**
    * {@inheritDoc}
@@ -78,6 +78,8 @@ public interface Transaction extends DatastoreReader, DatastoreWriter {
 
   /**
    * Rollback the transaction.
+   *
+   * @throws DatastoreServiceException if transaction was already committed
    */
   void rollback();
 
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index 7862681038c5..3babfd370e11 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -5,7 +5,7 @@
  * 
 {@code
  * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
- * KeyBuilder keyBuilder = datastore.newKeyBuilder(kind);
+ * KeyFactory keyBuilder = datastore.newKeyBuilder(kind);
  * Key key = keyBuilder.build(keyName);
  * Entity entity = datastore.get(key);
  * if (entity == null) {
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index fa7bae1da7fd..f0f9f8f4b52f 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -28,6 +28,7 @@ public interface Bucket {
   // that object can return its own meta-data, update its own meta-data, replace its content
   // via a stream or byteBuffer, read its content (via stream or ByteBuffer),...
   //void copy(String source, String bucket, String dest);
+  // Also consider read with an offset (and limit).
 
   void put(String name, ByteBuffer bytes);
 
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
index f88c5211ad93..10bd594b46a2 100644
--- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -4,19 +4,20 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.gcloud.ExceptionHandler.Interceptor;
+
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.channels.ClosedByInterruptException;
 import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Tests for {@link ExceptionHandler}.
  */
-@RunWith(JUnit4.class)
 public class ExceptionHandlerTest {
 
   @Test
@@ -92,6 +93,7 @@ private static  void assertInvalidCallable(Callable callable, ExceptionHan
     }
   }
 
+  @SuppressWarnings("serial")
   @Test
   public void testShouldTry() {
     ExceptionHandler handler = ExceptionHandler.builder().retryOn(IOException.class).build();
@@ -99,15 +101,52 @@ public void testShouldTry() {
     assertTrue(handler.shouldRetry(new ClosedByInterruptException()));
     assertFalse(handler.shouldRetry(new RuntimeException()));
 
-    handler = ExceptionHandler.builder()
+    ExceptionHandler.Builder builder = ExceptionHandler.builder()
         .retryOn(IOException.class, NullPointerException.class)
         .abortOn(RuntimeException.class, ClosedByInterruptException.class,
-            InterruptedException.class)
-        .build();
+            InterruptedException.class);
+
+    handler = builder.build();
     assertTrue(handler.shouldRetry(new IOException()));
     assertFalse(handler.shouldRetry(new ClosedByInterruptException()));
     assertFalse(handler.shouldRetry(new InterruptedException()));
     assertFalse(handler.shouldRetry(new RuntimeException()));
     assertTrue(handler.shouldRetry(new NullPointerException()));
+
+    final AtomicReference before = new AtomicReference<>(false);
+
+    Interceptor interceptor = new Interceptor() {
+      @Override
+      public boolean shouldRetry(Exception exception, boolean shouldRetry) {
+        return !shouldRetry;
+      }
+
+      @Override
+      public Boolean shouldRetry(Exception exception) {
+        return before.get();
+      }
+    };
+
+    builder.interceptor(interceptor);
+    handler = builder.build();
+    assertFalse(handler.shouldRetry(new IOException()));
+    assertFalse(handler.shouldRetry(new ClosedByInterruptException()));
+    assertFalse(handler.shouldRetry(new InterruptedException()));
+    assertFalse(handler.shouldRetry(new RuntimeException()));
+    assertFalse(handler.shouldRetry(new NullPointerException()));
+
+    before.set(true);
+    assertTrue(handler.shouldRetry(new IOException()));
+    assertTrue(handler.shouldRetry(new ClosedByInterruptException()));
+    assertTrue(handler.shouldRetry(new InterruptedException()));
+    assertTrue(handler.shouldRetry(new RuntimeException()));
+    assertTrue(handler.shouldRetry(new NullPointerException()));
+
+    before.set(null);
+    assertFalse(handler.shouldRetry(new IOException()));
+    assertTrue(handler.shouldRetry(new ClosedByInterruptException()));
+    assertTrue(handler.shouldRetry(new InterruptedException()));
+    assertTrue(handler.shouldRetry(new RuntimeException()));
+    assertFalse(handler.shouldRetry(new NullPointerException()));
   }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index fadd24147c67..92cd31cc282b 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -363,9 +363,9 @@ public void testRunStructuredQuery() {
 
   @Test
   public void testAllocateId() {
-    KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1);
-    PartialKey pk1 = keyBuilder.build();
-    Key key1 = keyBuilder.allocateIdAndBuild();
+    KeyFactory keyFactory = new KeyFactory(datastore, KIND1);
+    PartialKey pk1 = keyFactory.newKey();
+    Key key1 = keyFactory.allocateId();
     assertEquals(key1.dataset(), pk1.dataset());
     assertEquals(key1.namespace(), pk1.namespace());
     assertEquals(key1.ancestors(), pk1.ancestors());
@@ -385,11 +385,11 @@ public void testAllocateId() {
 
   @Test
   public void testAllocateIdArray() {
-    KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1);
-    PartialKey partialKey1 = keyBuilder.build();
-    PartialKey partialKey2 = keyBuilder.kind(KIND2).addAncestor(KIND1, 10).build();
-    Key key3 = keyBuilder.build("name");
-    Key key4 = keyBuilder.build(1);
+    KeyFactory keyFactory = new KeyFactory(datastore, KIND1);
+    PartialKey partialKey1 = keyFactory.newKey();
+    PartialKey partialKey2 = keyFactory.kind(KIND2).addAncestor(KIND1, 10).newKey();
+    Key key3 = keyFactory.newKey("name");
+    Key key4 = keyFactory.newKey(1);
     Iterator result =
         datastore.allocateId(partialKey1, partialKey2, key3, key4, partialKey1, key3);
     Map map = new HashMap<>();
@@ -525,12 +525,12 @@ public void testDelete() {
   }
 
   @Test
-  public void testNewKeyBuilder() {
-    KeyBuilder keyBuilder = datastore.newKeyBuilder(KIND1);
-    assertEquals(PARTIAL_KEY1, keyBuilder.build());
+  public void testKeyFactory() {
+    KeyFactory keyFactory = new KeyFactory(datastore, KIND1);
+    assertEquals(PARTIAL_KEY1, keyFactory.newKey());
     assertEquals(PartialKey.builder(PARTIAL_KEY1).kind(KIND2).build(),
-        datastore.newKeyBuilder(KIND2).build());
-    assertEquals(KEY1, keyBuilder.build("name"));
-    assertEquals(Key.builder(KEY1).id(2).build(), keyBuilder.build(2));
+        new KeyFactory(datastore, KIND2).newKey());
+    assertEquals(KEY1, keyFactory.newKey("name"));
+    assertEquals(Key.builder(KEY1).id(2).build(), keyFactory.newKey(2));
   }
 }

From 84e9d3219900e449d081345cb16cd0512aa97903 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 17 Dec 2014 17:48:06 -0800
Subject: [PATCH 062/732] handle deferred results for gets and change gets
 semantics (unordered).

---
 .../google/gcloud/datastore/BaseEntity.java   |  4 +-
 .../google/gcloud/datastore/BatchWriter.java  | 29 +++++-
 .../gcloud/datastore/DatastoreReader.java     |  9 +-
 .../datastore/DatastoreServiceImpl.java       | 69 ++++++++------
 .../com/google/gcloud/datastore/GqlQuery.java |  4 +-
 .../google/gcloud/datastore/QueryResult.java  |  2 +
 .../gcloud/datastore/QueryResultImpl.java     |  2 +-
 .../gcloud/datastore/StructuredQuery.java     |  4 +-
 .../google/gcloud/datastore/Transaction.java  | 10 +-
 .../gcloud/datastore/TransactionImpl.java     |  7 +-
 .../google/gcloud/datastore/package-info.java |  4 +-
 .../datastore/DatastoreServiceTest.java       | 92 +++++++++++--------
 12 files changed, 148 insertions(+), 88 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index d5497bb0b308..411967135b10 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -108,8 +108,8 @@ public B set(String name, List> values) {
       return self();
     }
 
-    public B set(String name, Value... value) {
-      properties.put(name, of(Arrays.asList(value)));
+    public B set(String name, Value value, Value... other) {
+      properties.put(name, of(value, other));
       return self();
     }
 
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java
index 99aeafabd3d4..9fadd9eca6a9 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java
@@ -20,22 +20,45 @@ public interface BatchWriter extends DatastoreWriter {
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceException if a given entity already added to this batch
+   * This operation will be converted to {@link #put} operation for entities that were already
+   *     marked for deletion in this batch.
+   * @throws DatastoreServiceException if a given entity already added to this batch or if batch
+   *     is no longer active
    */
   @Override
   void add(Entity... entity);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceException if an entity is marked for deletion in this batch
+   * This operation will be converted to {@link #put} operation for entities that were already
+   *     added or put in this batch.
+   * @throws DatastoreServiceException if an entity is marked for deletion in this batch or if
+   *     batch is no longer active
    */
   @Override
   void update(Entity... entity);
 
+  /**
+   * {@inheritDoc}
+   * This operation will also remove from this batch any prior writes for entities with the same
+   *     keys.
+   * @throws DatastoreServiceException if batch is no longer active
+   */
+  @Override
+  public void delete(Key... key);
+
+  /**
+   * {@inheritDoc}
+   * This operation will also remove from this batch any prior writes for the same entities.
+   * @throws DatastoreServiceException if batch is no longer active
+   */
+  @Override
+  public void put(Entity... entity);
+
   /**
    * Submit the batch to the Datastore.
    *
-   * @throws DatastoreServiceException if there was any failure.
+   * @throws DatastoreServiceException if there was any failure or if batch is not longer active
    */
   void submit();
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index ec7052e414e0..9fc1f0c5b9b5 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -15,8 +15,11 @@ public interface DatastoreReader {
   Entity get(Key key);
 
   /**
-   * Returns an {@link Entity} for each given {@link Key} or {@code null} if does not exists
-   * ordered by input.
+   * Returns an {@link Entity} for each given {@link Key} that exists in the Datastore.
+   * The order of the result is unspecified.
+   * Results are loaded lazily therefore it is possible to get a DatastoreServiceException
+   * from the returned {@code Iterator} {@link Iterator#hasNext hasNext} or
+   * {@link Iterator#next next} calls.
    *
    * @throws DatastoreServiceException upon failure.
    */
@@ -27,5 +30,5 @@ public interface DatastoreReader {
    *
    * @throws DatastoreServiceException upon failure.
    */
-   QueryResult runQuery(Query query);
+   QueryResult run(Query query);
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 7af34773c3b8..7d7f7c8191f6 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -7,11 +7,9 @@
 import com.google.protobuf.ByteString;
 
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
-import java.util.Map;
 
 
 final class DatastoreServiceImpl implements DatastoreService {
@@ -43,15 +41,15 @@ public Transaction newTransaction(TransactionOption... transactionOption) {
   }
 
   @Override
-  public  QueryResult runQuery(Query query) {
-    return runQuery(null, query);
+  public  QueryResult run(Query query) {
+    return run(null, query);
   }
 
-   QueryResult runQuery(DatastoreV1.ReadOptions readOptionsPb, Query query) {
+   QueryResult run(DatastoreV1.ReadOptions readOptionsPb, Query query) {
     return new QueryResultImpl<>(this, readOptionsPb, query);
   }
 
-  DatastoreV1.RunQueryResponse runQuery(DatastoreV1.RunQueryRequest requestPb) {
+  DatastoreV1.RunQueryResponse run(DatastoreV1.RunQueryRequest requestPb) {
     try {
       return datastore.runQuery(requestPb);
     } catch (DatastoreException e) {
@@ -99,7 +97,8 @@ private PartialKey trimNameOrId(PartialKey key) {
 
   @Override
   public Entity get(Key key) {
-    return get(key, EMPTY_KEY_ARRAY).next();
+    Iterator iter = get(key, EMPTY_KEY_ARRAY);
+    return iter.hasNext() ? iter.next() : null;
   }
 
   @Override
@@ -108,7 +107,7 @@ public Iterator get(Key key, Key... others) {
   }
 
   Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key key, final Key... others) {
-    DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
+    final DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
     if (readOptionsPb != null) {
       requestPb.setReadOptions(readOptionsPb);
     }
@@ -118,30 +117,44 @@ Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key key, final
     for (Key k : dedupKeys) {
       requestPb.addKey(k.toPb());
     }
-    try {
-      DatastoreV1.LookupResponse responsePb = datastore.lookup(requestPb.build());
-      final Map result = new HashMap<>();
-      for (DatastoreV1.EntityResult entityResultPb : responsePb.getFoundList()) {
-        Entity entity = Entity.fromPb(entityResultPb.getEntity());
-        result.put(entity.key(), entity);
+    return new ResultsIterator(requestPb);
+  }
+
+  final class ResultsIterator extends AbstractIterator {
+
+    private final DatastoreV1.LookupRequest.Builder requestPb;
+    Iterator iter;
+
+    ResultsIterator(DatastoreV1.LookupRequest.Builder requestPb) {
+      this.requestPb = requestPb;
+      loadResults();
+    }
+
+    private void loadResults() {
+      try {
+        DatastoreV1.LookupResponse responsePb = datastore.lookup(requestPb.build());
+        iter = responsePb.getFoundList().iterator();
+        requestPb.clearKey();
+        if (responsePb.getDeferredCount() > 0) {
+          requestPb.addAllKey(responsePb.getDeferredList());
+        }
+      } catch (DatastoreException e) {
+        throw DatastoreServiceException.translateAndThrow(e);
       }
-      return new AbstractIterator() {
-        int index = -2;
+    }
 
-        @Override
-        protected Entity computeNext() {
-          ++index;
-          if (index < 0) {
-            return result.get(key);
-          }
-          if (index < others.length) {
-            return result.get(others[index]);
-          }
+    @Override
+    protected Entity computeNext() {
+      if (iter.hasNext()) {
+        return Entity.fromPb(iter.next().getEntity());
+      }
+      while (!iter.hasNext()) {
+        if (requestPb.getKeyCount() == 0) {
           return endOfData();
         }
-      };
-    } catch (DatastoreException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+        loadResults();
+      }
+      return Entity.fromPb(iter.next().getEntity());
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index f8769d8ac0ae..df24b787b3fb 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -29,7 +29,7 @@
  * 
When the type of the results is known the preferred usage would be:
  * 
 {@code
  *   Query query = GqlQuery.builder(ResultClass.full(), "select * from kind").build();
- *   QueryResult results = datastore.runQuery(query);
+ *   QueryResult results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -39,7 +39,7 @@
  * 
When the type of the results is unknown you can use this approach:
  * 
 {@code
  *   Query query = GqlQuery.builder("select __key__ from kind").build();
- *   QueryResult results = datastore.runQuery(query);
+ *   QueryResult results = datastore.run(query);
  *   if (Key.class.isAssignableFrom(results.resultClass())) {
  *     QueryResult keys = (QueryResult) results;
  *     while (keys.hasNext()) {
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index 4c69f6b73eaa..aa615b755f19 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -6,6 +6,8 @@
  * The result of a Google Cloud Datastore query submission.
  * When result is not typed it is possible to cast it to its appropriate type according to
  * the {@link #resultClass} value.
+ * Results are loaded lazily therefore it is possible to get a DatastoreServiceException
+ * upon {@link #hasNext} or {@link #next} calls.
  *
  * @param V the type of the results value.
  */
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index eecf6a196e6c..ef18701bf4c1 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -55,7 +55,7 @@ private DatastoreV1.QueryResultBatch sendRequest() {
     }
     requestPb.setPartitionId(partitionIdPb);
     query.populatePb(requestPb);
-    resultPb = datastore.runQuery(requestPb.build()).getBatch();
+    resultPb = datastore.run(requestPb.build()).getBatch();
     entityResultPbIter = resultPb.getEntityResultList().iterator();
     // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
     type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index f9a733d1a529..328d8a896d8c 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -31,7 +31,7 @@
  * 
A simple query that returns all entities for a specific kind
  * 
 {@code
  *   StructuredQuery query = StructuredQuery.builder().kind(kind).build();
- *   QueryResult results = datastore.runQuery(query);
+ *   QueryResult results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -49,7 +49,7 @@
  *       .orderBy(OrderBy.asc("age"))
  *       .limit(10)
  *       .build();
- *   QueryResult results = datastore.runQuery(query);
+ *   QueryResult results = datastore.run(query);
  *   ...
  * } 

  *
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index 2a9267cf6d41..06c4e5a45997 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -41,7 +41,7 @@ public interface Transaction extends DatastoreReaderWriter {
    * to fail if entity was changed by others after it was seen by this transaction) but any
    * write changes in this transaction will not be reflected by the returned entity.
    *
-   * @throws DatastoreServiceException upon failure.
+   * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
   Entity get(Key key);
@@ -52,7 +52,7 @@ public interface Transaction extends DatastoreReaderWriter {
    * to fail if any of the entities was changed by others after they were seen by this transaction)
    * but any write changes in this transaction will not be reflected by the returned entities.
    *
-   * @throws DatastoreServiceException upon failure.
+   * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
   Iterator get(Key key, Key... others);
@@ -64,15 +64,15 @@ public interface Transaction extends DatastoreReaderWriter {
    * query was performed) but any write changes in this transaction will not be reflected by
    * the result.
    *
-   * @throws DatastoreServiceException upon failure.
+   * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
-   QueryResult runQuery(Query query);
+   QueryResult run(Query query);
 
   /**
    * Commit the transaction.
    *
-   * @throws DatastoreServiceException if could not commit the transaction
+   * @throws DatastoreServiceException if could not commit the transaction or if no longer active
    */
   void commit();
 
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index ee0dc31685ed..a2e15b311689 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -29,7 +29,8 @@ public final class TransactionImpl extends BatchWriterImpl implements Transactio
 
   @Override
   public Entity get(Key key) {
-    return get(key, DatastoreServiceImpl.EMPTY_KEY_ARRAY).next();
+    Iterator iter = get(key, DatastoreServiceImpl.EMPTY_KEY_ARRAY);
+    return iter.hasNext() ? iter.next() : null;
   }
 
   @Override
@@ -41,11 +42,11 @@ public Iterator get(Key key, Key... others) {
   }
 
   @Override
-  public  QueryResult runQuery(Query query) {
+  public  QueryResult run(Query query) {
     checkActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
-    return datastore.runQuery(readOptionsPb.build(), query);
+    return datastore.run(readOptionsPb.build(), query);
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index 3babfd370e11..ea3e92b7386b 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -5,8 +5,8 @@
  * 
 {@code
  * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
- * KeyFactory keyBuilder = datastore.newKeyBuilder(kind);
- * Key key = keyBuilder.build(keyName);
+ * KeyFactory keyFactory = new KeyFactory(datastore, kind);
+ * Key key = keyFactory.newKey(keyName);
  * Entity entity = datastore.get(key);
  * if (entity == null) {
  *   entity = Entity.builder(key)
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 92cd31cc282b..a7c6626d6b92 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -8,6 +8,7 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.common.collect.Maps;
 import com.google.gcloud.datastore.Query.ResultClass;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
@@ -16,9 +17,11 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 public class DatastoreServiceTest {
@@ -93,11 +96,11 @@ public void testNewTransactionCommit() {
     transaction.delete(KEY1);
     transaction.commit();
 
-    Iterator iter = datastore.get(KEY1, KEY2, KEY3);
-    assertNull(iter.next());
-    assertEquals(entity2, iter.next());
-    assertEquals(ENTITY3, iter.next());
-    assertFalse(iter.hasNext());
+    List list = fetch(KEY1, KEY2, KEY3);
+    assertNull(list.get(0));
+    assertEquals(entity2, list.get(1));
+    assertEquals(ENTITY3, list.get(2));
+    assertEquals(3, list.size());
 
     try {
       transaction.commit();
@@ -143,7 +146,7 @@ public void testTransactionWithQuery() {
     Query query =
         StructuredQuery.builder().kind(KIND2).filter(PropertyFilter.hasAncestor(KEY2)).build();
     Transaction transaction = datastore.newTransaction();
-    QueryResult results = transaction.runQuery(query);
+    QueryResult results = transaction.run(query);
     assertEquals(ENTITY2, results.next());
     assertFalse(results.hasNext());
     transaction.add(ENTITY3);
@@ -151,7 +154,7 @@ public void testTransactionWithQuery() {
     assertEquals(ENTITY3, datastore.get(KEY3));
 
     transaction = datastore.newTransaction();
-    results = transaction.runQuery(query);
+    results = transaction.run(query);
     assertEquals(ENTITY2, results.next());
     transaction.delete(ENTITY3.key());
     // update entity2 during the transaction
@@ -185,11 +188,11 @@ public void testNewTransactionRollback() {
 
     verifyNotUsable(transaction);
 
-    Iterator iter = datastore.get(KEY1, KEY2, KEY3);
-    assertEquals(ENTITY1, iter.next());
-    assertEquals(ENTITY2, iter.next());
-    assertNull(iter.next());
-    assertFalse(iter.hasNext());
+    List list = fetch(KEY1, KEY2, KEY3);
+    assertEquals(ENTITY1, list.get(0));
+    assertEquals(ENTITY2, list.get(1));
+    assertNull(list.get(2));
+    assertEquals(3, list.size());
   }
 
   private void verifyNotUsable(DatastoreWriter writer) {
@@ -232,7 +235,7 @@ public void testNewBatchWriter() {
     batchWriter.add(entity4, entity5);
     batchWriter.put(ENTITY3, entity1, entity2);
     batchWriter.submit();
-    Iterator entities = datastore.get(KEY1, KEY2, KEY3, entity4.key(), entity5.key());
+    Iterator entities = fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(entity1, entities.next());
     assertEquals(entity2, entities.next());
     assertEquals(ENTITY3, entities.next());
@@ -252,7 +255,7 @@ public void testNewBatchWriter() {
     batchWriter.delete(entity4.key(), entity5.key());
     batchWriter.update(ENTITY1, ENTITY2, ENTITY3);
     batchWriter.submit();
-    entities = datastore.get(KEY1, KEY2, KEY3, entity4.key(), entity5.key());
+    entities = fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(ENTITY1, entities.next());
     assertEquals(ENTITY2, entities.next());
     assertEquals(ENTITY3, entities.next());
@@ -270,7 +273,7 @@ public void testNewBatchWriter() {
   @Test
   public void testRunGqlQueryNoCasting() {
     Query query1 = GqlQuery.builder(ResultClass.full(), "select * from " + KIND1).build();
-    QueryResult results1 = datastore.runQuery(query1);
+    QueryResult results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
@@ -278,7 +281,7 @@ public void testRunGqlQueryNoCasting() {
     datastore.put(ENTITY3);
     Query query2 =  GqlQuery.builder(
         ResultClass.full(), "select * from " + KIND2 + " order by __key__").build();
-    QueryResult results2 = datastore.runQuery(query2);
+    QueryResult results2 = datastore.run(query2);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY2, results2.next());
     assertTrue(results2.hasNext());
@@ -286,19 +289,19 @@ public void testRunGqlQueryNoCasting() {
     assertFalse(results2.hasNext());
 
     query1 = GqlQuery.builder(ResultClass.full(), "select * from bla").build();
-    results1 = datastore.runQuery(query1);
+    results1 = datastore.run(query1);
     assertFalse(results1.hasNext());
 
     Query keyOnlyQuery =
         GqlQuery.builder(ResultClass.keyOnly(), "select __key__ from " + KIND1).build();
-    QueryResult keyOnlyResults = datastore.runQuery(keyOnlyQuery);
+    QueryResult keyOnlyResults = datastore.run(keyOnlyQuery);
     assertTrue(keyOnlyResults.hasNext());
     assertEquals(KEY1, keyOnlyResults.next());
     assertFalse(keyOnlyResults.hasNext());
 
     Query projectionQuery = GqlQuery.builder(
         ResultClass.projection(), "select str from " + KIND1).build();
-    QueryResult projectionResult = datastore.runQuery(projectionQuery);
+    QueryResult projectionResult = datastore.run(projectionQuery);
     assertTrue(projectionResult.hasNext());
     PartialEntity partialEntity = projectionResult.next();
     assertEquals("str", partialEntity.getString("str"));
@@ -310,13 +313,13 @@ public void testRunGqlQueryNoCasting() {
   public void testRunGqlQueryWithCasting() {
     @SuppressWarnings("unchecked")
     Query query1 = (Query) GqlQuery.builder("select * from " + KIND1).build();
-    QueryResult results1 = datastore.runQuery(query1);
+    QueryResult results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
     Query query2 = GqlQuery.builder("select * from " + KIND1).build();
-    QueryResult results2 = datastore.runQuery(query2);
+    QueryResult results2 = datastore.run(query2);
     assertEquals(Entity.class, results2.resultClass());
     @SuppressWarnings("unchecked")
     QueryResult results3 = (QueryResult) results2;
@@ -329,13 +332,13 @@ public void testRunGqlQueryWithCasting() {
   public void testRunStructuredQuery() {
     StructuredQuery query =
         StructuredQuery.builder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
-    QueryResult results1 = datastore.runQuery(query);
+    QueryResult results1 = datastore.run(query);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
     StructuredQuery keyOnlyQuery =  StructuredQuery.keyOnlyBuilder().kind(KIND1).build();
-    QueryResult results2 = datastore.runQuery(keyOnlyQuery);
+    QueryResult results2 = datastore.run(keyOnlyQuery);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY1.key(), results2.next());
     assertFalse(results2.hasNext());
@@ -349,7 +352,7 @@ public void testRunStructuredQuery() {
         .limit(10)
         .build();
 
-    QueryResult results3 = datastore.runQuery(projectionQuery);
+    QueryResult results3 = datastore.run(projectionQuery);
     assertTrue(results3.hasNext());
     PartialEntity entity = results3.next();
     assertEquals(ENTITY2.key(), entity.key());
@@ -428,7 +431,7 @@ public void testGet() {
   public void testGetArray() {
     datastore.put(ENTITY3);
     Iterator result =
-        datastore.get(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3);
+        fetch(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3).iterator();
     assertEquals(ENTITY1, result.next());
     assertNull(result.next());
     assertEquals(ENTITY2, result.next());
@@ -453,12 +456,27 @@ public void testGetArray() {
     assertFalse(result.hasNext());
   }
 
+  public List fetch(Key key, Key... others) {
+    Iterator entities = datastore.get(key, others);
+    Map map = Maps.newHashMapWithExpectedSize(1 + others.length);
+    while (entities.hasNext()) {
+      Entity entity = entities.next();
+      map.put(entity.key(), entity);
+    }
+    List list = new ArrayList<>(1 + others.length);
+    list.add(map.get(key));
+    for (Key other : others) {
+      list.add(map.get(other));
+    }
+    return list;
+  }
+
   @Test
   public void testAdd() {
-    Iterator keys = datastore.get(ENTITY1.key(), ENTITY3.key());
-    assertEquals(ENTITY1, keys.next());
-    assertNull(keys.next());
-    assertFalse(keys.hasNext());
+    List keys = fetch(ENTITY1.key(), ENTITY3.key());
+    assertEquals(ENTITY1, keys.get(0));
+    assertNull(keys.get(1));
+    assertEquals(2, keys.size());
 
     try {
       datastore.add(ENTITY1);
@@ -472,10 +490,10 @@ public void testAdd() {
 
   @Test
   public void testUpdate() {
-    Iterator keys = datastore.get(ENTITY1.key(), ENTITY3.key());
-    assertEquals(ENTITY1, keys.next());
-    assertNull(keys.next());
-    assertFalse(keys.hasNext());
+    List keys = fetch(ENTITY1.key(), ENTITY3.key());
+    assertEquals(ENTITY1, keys.get(0));
+    assertNull(keys.get(1));
+    assertEquals(2, keys.size());
 
     try {
       datastore.update(ENTITY3);
@@ -493,7 +511,7 @@ public void testUpdate() {
 
   @Test
   public void testPut() {
-    Iterator keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
+    Iterator keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
@@ -502,7 +520,7 @@ public void testPut() {
     Entity entity2 = Entity.builder(ENTITY2).clear().set("bla", new NullValue()).build();
     assertNotEquals(ENTITY2, entity2);
     datastore.put(ENTITY3, ENTITY1, entity2);
-    keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
+    keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(entity2, keys.next());
     assertEquals(ENTITY3, keys.next());
@@ -511,13 +529,13 @@ public void testPut() {
 
   @Test
   public void testDelete() {
-    Iterator keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
+    Iterator keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
     assertFalse(keys.hasNext());
     datastore.delete(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
-    keys = datastore.get(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
+    keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertNull(keys.next());
     assertNull(keys.next());
     assertNull(keys.next());

From d997837f3fdfe2611e023d906578bcc369e1d1cc Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 18 Dec 2014 10:03:56 -0800
Subject: [PATCH 063/732] minor change

---
 src/main/java/com/google/gcloud/datastore/BaseEntity.java       | 1 -
 .../java/com/google/gcloud/datastore/DatastoreServiceImpl.java  | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 411967135b10..0ec36a2f0211 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -15,7 +15,6 @@
 import com.google.common.collect.ImmutableSortedMap;
 import com.google.gcloud.datastore.Value.Type;
 
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 7d7f7c8191f6..91bfa0137b57 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -107,7 +107,7 @@ public Iterator get(Key key, Key... others) {
   }
 
   Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key key, final Key... others) {
-    final DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
+    DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
     if (readOptionsPb != null) {
       requestPb.setReadOptions(readOptionsPb);
     }

From 9048e0b8f6fc36bcec9af2387af1755a593b320e Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 18 Dec 2014 17:03:46 -0800
Subject: [PATCH 064/732] change projection query to return projection entity

---
 .../gcloud/datastore/BatchWriteOption.java    |   3 +-
 .../gcloud/datastore/DatastoreReader.java     |   6 +-
 .../com/google/gcloud/datastore/GqlQuery.java |  20 +-
 .../gcloud/datastore/PartialEntity.java       |   2 +-
 .../gcloud/datastore/ProjectionEntity.java    | 128 +++++++++++++
 .../com/google/gcloud/datastore/Query.java    | 177 +++++++++---------
 .../google/gcloud/datastore/QueryResult.java  |   4 +-
 .../gcloud/datastore/QueryResultImpl.java     |  36 ++--
 .../gcloud/datastore/StructuredQuery.java     |  39 ++--
 .../gcloud/datastore/TransactionImpl.java     |  19 +-
 .../gcloud/datastore/TransactionOption.java   |  44 ++++-
 .../datastore/DatastoreServiceTest.java       |  59 +++---
 .../gcloud/datastore/SerializationTest.java   |  10 +-
 13 files changed, 364 insertions(+), 183 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/ProjectionEntity.java

diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java
index 2fea6e95d893..f2bf33266252 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java
@@ -2,10 +2,9 @@
 
 import com.google.common.collect.ImmutableMap;
 
-import java.io.Serializable;
 import java.util.Map;
 
-public abstract class BatchWriteOption implements Serializable {
+public abstract class BatchWriteOption implements java.io.Serializable {
 
   private static final long serialVersionUID = -3932758377282659839L;
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index 9fc1f0c5b9b5..ea33c0104388 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -17,9 +17,9 @@ public interface DatastoreReader {
   /**
    * Returns an {@link Entity} for each given {@link Key} that exists in the Datastore.
    * The order of the result is unspecified.
-   * Results are loaded lazily therefore it is possible to get a DatastoreServiceException
-   * from the returned {@code Iterator} {@link Iterator#hasNext hasNext} or
-   * {@link Iterator#next next} calls.
+   * Results are loaded lazily therefore it is possible to get a {@code DatastoreServiceException}
+   * from the returned {@code Iterator}'s {@link Iterator#hasNext hasNext} or
+   * {@link Iterator#next next} methods.
    *
    * @throws DatastoreServiceException upon failure.
    */
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index df24b787b3fb..c80cb4a77f14 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -28,7 +28,7 @@
  *
  * 
When the type of the results is known the preferred usage would be:
  * 
 {@code
- *   Query query = GqlQuery.builder(ResultClass.full(), "select * from kind").build();
+ *   Query query = GqlQuery.builder(Query.Type.FULL, "select * from kind").build();
  *   QueryResult results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
@@ -142,15 +142,15 @@ static Argument fromPb(DatastoreV1.GqlQueryArg argPb) {
    */
   public static final class Builder {
 
-    private final ResultClass resultClass;
+    private final Type type;
     private String namespace;
     private String queryString;
     private boolean allowLiteral;
     private Map nameArgs = new TreeMap<>();
     private List numberArgs = new LinkedList<>();
 
-    Builder(ResultClass resultClass, String query) {
-      this.resultClass = checkNotNull(resultClass);
+    Builder(Type type, String query) {
+      this.type = checkNotNull(type);
       queryString = checkNotNull(query);
     }
 
@@ -294,7 +294,7 @@ private static Argument toArgument(String name, Value.BuilderFactory builderFact
   }
 
   private GqlQuery(Builder builder) {
-    super(builder.resultClass, builder.namespace);
+    super(builder.type, builder.namespace);
     queryString = builder.queryString;
     allowLiteral = builder.allowLiteral;
     nameArgs = ImmutableList.copyOf(builder.nameArgs.values());
@@ -378,12 +378,12 @@ protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
   }
 
   @Override
-  protected Object fromPb(ResultClass resultType, String namespace, byte[] bytesPb)
+  protected Object fromPb(Type resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
     return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
   }
 
-  static  GqlQuery fromPb(ResultClass resultType, String namespace,
+  static  GqlQuery fromPb(Type resultType, String namespace,
       DatastoreV1.GqlQuery queryPb) {
     Builder builder = new Builder<>(resultType, queryPb.getQueryString());
     builder.namespace(namespace);
@@ -407,7 +407,7 @@ static  GqlQuery fromPb(ResultClass resultType, String namespace,
    * @see GQL Reference
    */
   public static GqlQuery.Builder builder(String gql) {
-    return builder(ResultClass.unknown(), gql);
+    return builder(Type.UNKNOWN, gql);
   }
 
   /**
@@ -415,7 +415,7 @@ public static GqlQuery.Builder builder(String gql) {
    *
    * @see GQL Reference
    */
-  public static  GqlQuery.Builder builder(ResultClass resultClass, String gql) {
-    return new GqlQuery.Builder<>(resultClass, gql);
+  public static  GqlQuery.Builder builder(Type type, String gql) {
+    return new GqlQuery.Builder<>(type, gql);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index 9edefac1ca34..df520d3f48b9 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -48,7 +48,7 @@ protected PartialEntity(PartialKey key, ImmutableSortedMap> pro
   }
 
   /**
-   * Returns a new {@link #Entity} with the same properties as this one and
+   * Returns a new {@link Entity} with the same properties as this one and
    * with the given {@code key}.
    */
   public Entity toEntity(Key key) {
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
new file mode 100644
index 000000000000..6b219ce0968c
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -0,0 +1,128 @@
+package com.google.gcloud.datastore;
+
+import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+import java.util.Objects;
+
+/**
+ * A projection entity is a result of a Google Cloud Datastore projection query.
+ * A projection entity holds one or more properties, represented by a name (as {@link String})
+ * and a value (as {@link Value}), and may have a {@link Key}.
+ *
+ * @see Google Cloud Datastore projection queries
+ * @see Google Cloud Datastore Entities, Properties, and Keys
+ */
+public final class ProjectionEntity extends BaseEntity {
+
+  private static final long serialVersionUID = 432961565733066915L;
+
+  private final Key key;
+
+  static final class Builder extends BaseEntity.Builder {
+
+    private Key key;
+
+    private Builder() {
+    }
+
+    private Builder(ProjectionEntity entity) {
+      super(entity);
+      key = entity.key();
+    }
+
+    public Builder key(Key key) {
+      this.key = key;
+      return this;
+    }
+
+    @Override
+    public ProjectionEntity build() {
+      return new ProjectionEntity(key, ImmutableSortedMap.copyOf(properties));
+    }
+  }
+
+  ProjectionEntity(Key key, ImmutableSortedMap> properties) {
+    super(properties);
+    this.key = key;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(key, properties());
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+    if (!(obj instanceof ProjectionEntity)) {
+      return false;
+    }
+    ProjectionEntity other = (ProjectionEntity) obj;
+    return Objects.equals(key, other.key)
+        && Objects.equals(properties(), other.properties());
+  }
+
+  public boolean hasKey() {
+    return key() != null;
+  }
+
+  /**
+   * Returns the associated {@link Key} or null if it does not have one.
+   */
+  public Key key() {
+    return key;
+  }
+
+  @Override
+  public DateTime getDateTime(String name) {
+    Value value = getValue(name);
+    if (value.hasMeaning() && value.meaning() == 18 && value instanceof LongValue) {
+      return new DateTime(getLong(name));
+    }
+    return ((DateTimeValue) value).get();
+  }
+
+  @Override
+  public Blob getBlob(String name) {
+    Value value = getValue(name);
+    if (value.hasMeaning() && value.meaning() == 18 && value instanceof StringValue) {
+      return new Blob(ByteString.copyFromUtf8(getString(name)), false);
+    }
+    return ((BlobValue) value).get();
+  }
+
+  @Override
+  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+    return fromPb(DatastoreV1.Entity.parseFrom(bytesPb));
+  }
+
+  static ProjectionEntity fromPb(DatastoreV1.Entity entityPb) {
+    ImmutableSortedMap.Builder> properties =
+        ImmutableSortedMap.naturalOrder();
+    for (DatastoreV1.Property property : entityPb.getPropertyList()) {
+      properties.put(property.getName(), Value.fromPb(property.getValue()));
+    }
+    Key key = null;
+    if (entityPb.hasKey()) {
+      key = Key.fromPb(entityPb.getKey());
+    }
+    return new ProjectionEntity(key, properties.build());
+  }
+
+
+  public static Builder builder(ProjectionEntity copyFrom) {
+    return new Builder(copyFrom);
+  }
+
+  @Override
+  protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {
+    if (key != null) {
+      entityPb.setKey(key.toPb());
+    }
+  }
+}
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 03df75c00312..c68e3246f62f 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -5,9 +5,12 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.collect.Maps;
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 
+import java.util.EnumMap;
+
 
 /**
  * A Google Cloud Datastore query.
@@ -20,31 +23,84 @@ public abstract class Query extends Serializable {
 
   private static final long serialVersionUID = -2748141759901313101L;
 
-  private final ResultClass resultClass;
+  private final Type type;
   private final String namespace;
 
-  public static class ResultClass implements java.io.Serializable {
+  /**
+   * This class represents the expected type of the result.
+   *   FULL: A complete {@link Entity}.
+   *   PROJECTION: A partial entity, represented by {@link PartialEntity}.
+   *   KEY_ONLY: An entity's {@link Key}.
+   */
+  public abstract static class Type implements java.io.Serializable {
 
     private static final long serialVersionUID = 2104157695425806623L;
-    private static final ResultClass UNKNOWN = new ResultClass<>(Object.class);
-    private static final ResultClass FULL = new ResultClass<>(Entity.class);
-    private static final ResultClass KEY_ONLY = new ResultClass<>(Key.class);
-    private static final ResultClass PROJECTION =
-        new ResultClass<>(PartialEntity.class);
+    private static final EnumMap>
+        PB_TO_INSTANCE = Maps.newEnumMap(DatastoreV1.EntityResult.ResultType.class);
+
+    static final Type UNKNOWN = new Type(null, Object.class) {
+
+      private static final long serialVersionUID = 1602329532153860907L;
+
+      @Override protected Object convert(DatastoreV1.Entity entityPb) {
+        if (entityPb.getPropertyCount() == 0) {
+          if (!entityPb.hasKey()) {
+            return null;
+          }
+          return Key.fromPb(entityPb.getKey());
+        }
+        return ProjectionEntity.fromPb(entityPb);
+      }
+    };
+
+    public static final Type FULL =
+        new Type(DatastoreV1.EntityResult.ResultType.FULL, Entity.class) {
+
+      private static final long serialVersionUID = 7712959777507168274L;
+
+      @Override protected Entity convert(DatastoreV1.Entity entityPb) {
+        return Entity.fromPb(entityPb);
+      }
+    };
+
+    public static final Type KEY_ONLY =
+        new Type(DatastoreV1.EntityResult.ResultType.KEY_ONLY, Key.class) {
+
+      private static final long serialVersionUID = -8514289244104446252L;
+
+      @Override protected Key convert(DatastoreV1.Entity entityPb) {
+        return Key.fromPb(entityPb.getKey());
+      }
+    };
 
-    private final Class value;
+    public static final Type PROJECTION = new Type(
+        DatastoreV1.EntityResult.ResultType.PROJECTION, ProjectionEntity.class) {
 
-    private ResultClass(Class value) {
-      this.value = checkNotNull(value);
+          private static final long serialVersionUID = -7591409419690650246L;
+
+          @Override protected ProjectionEntity convert(DatastoreV1.Entity entityPb) {
+            return ProjectionEntity.fromPb(entityPb);
+          }
+    };
+
+    private final Class resultClass;
+    private final DatastoreV1.EntityResult.ResultType resultType;
+
+    private Type(DatastoreV1.EntityResult.ResultType typePb, Class resultClass) {
+      this.resultType = typePb;
+      this.resultClass = checkNotNull(resultClass);
+      if (typePb != null) {
+        PB_TO_INSTANCE.put(typePb, this);
+      }
     }
 
-    public Class value() {
-      return value;
+    public Class resultClass() {
+      return resultClass;
     }
 
     @Override
     public int hashCode() {
-      return value.hashCode();
+      return resultClass.hashCode();
     }
 
     @Override
@@ -52,102 +108,39 @@ public boolean equals(Object obj) {
       if (obj == this) {
         return true;
       }
-      if (!(obj instanceof ResultClass)) {
+      if (!(obj instanceof Type)) {
         return false;
       }
-      ResultClass other = (ResultClass) obj;
-      return value.equals(other.value);
+      Type other = (Type) obj;
+      return resultClass.equals(other.resultClass);
     }
 
     @Override
     public String toString() {
       ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
-      toStringHelper.add("value", value);
+      toStringHelper.add("resultType", resultType);
+      toStringHelper.add("resultClass", resultClass);
       return toStringHelper.toString();
     }
 
-    boolean isAssignableFrom(ResultClass resultClass) {
-      return value.isAssignableFrom(resultClass.value);
+    boolean isAssignableFrom(Type otherType) {
+      return resultClass.isAssignableFrom(otherType.resultClass);
     }
 
-    static ResultClass unknown() {
-      return UNKNOWN;
-    }
+    protected abstract V convert(DatastoreV1.Entity value);
 
-    public static ResultClass full() {
-      return FULL;
+    static Type fromPb(DatastoreV1.EntityResult.ResultType typePb) {
+      return MoreObjects.firstNonNull(PB_TO_INSTANCE.get(typePb), UNKNOWN);
     }
-
-    public static ResultClass projection() {
-      return PROJECTION;
-    }
-
-    public static ResultClass keyOnly() {
-      return KEY_ONLY;
-    }
-  }
-
-  /**
-   * Possible results types are:
-   *   FULL: A complete {@link Entity}.
-   *   PROJECTION: A partial entity, represented by {@link PartialEntity}.
-   *   KEY_ONLY: An entity's {@link Key}.
-   */
-  public static enum Type {
-
-    FULL {
-      @Override
-      @SuppressWarnings("unchecked")
-      Entity convert(DatastoreV1.Entity value) {
-        return Entity.fromPb(value);
-      }
-
-      @Override
-      ResultClass resultClass() {
-        return ResultClass.full();
-      }
-    },
-
-    PROJECTION  {
-
-      @Override
-      @SuppressWarnings("unchecked")
-      PartialEntity convert(DatastoreV1.Entity value) {
-        return PartialEntity.fromPb(value);
-      }
-
-      @Override
-      ResultClass resultClass() {
-        return ResultClass.projection();
-      }
-    },
-
-    KEY_ONLY {
-
-      @Override
-      @SuppressWarnings("unchecked")
-      Key convert(DatastoreV1.Entity value) {
-        return Key.fromPb(value.getKey());
-      }
-
-      @Override
-      ResultClass resultClass() {
-        return ResultClass.keyOnly();
-      }
-    };
-
-    abstract  V convert(DatastoreV1.Entity value);
-
-    abstract ResultClass resultClass();
   }
 
-  Query(ResultClass resultClass, String namespace) {
-    this.resultClass = checkNotNull(resultClass);
+  Query(Type type, String namespace) {
+    this.type = checkNotNull(type);
     this.namespace = namespace;
   }
 
-  ResultClass resultClass() {
-    return resultClass;
+  Type type() {
+    return type;
   }
 
   public String namespace() {
@@ -164,10 +157,10 @@ public String toString() {
 
   @Override
   protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    return fromPb(resultClass, namespace, bytesPb);
+    return fromPb(type, namespace, bytesPb);
   }
 
-  protected abstract Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
+  protected abstract Object fromPb(Type type, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException;
 
   protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb);
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index aa615b755f19..3dab5967e81d 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -6,8 +6,8 @@
  * The result of a Google Cloud Datastore query submission.
  * When result is not typed it is possible to cast it to its appropriate type according to
  * the {@link #resultClass} value.
- * Results are loaded lazily therefore it is possible to get a DatastoreServiceException
- * upon {@link #hasNext} or {@link #next} calls.
+ * Results are loaded lazily therefore it is possible to get a {@code DatastoreServiceException}
+ * upon {@link Iterator#hasNext hasNext} or {@link Iterator#next next} calls.
  *
  * @param V the type of the results value.
  */
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index ef18701bf4c1..808d9e8d7662 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -1,42 +1,31 @@
 package com.google.gcloud.datastore;
 
-import com.google.api.client.repackaged.com.google.common.base.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.AbstractIterator;
-import com.google.common.collect.ImmutableMap;
+import com.google.gcloud.datastore.Query.Type;
 
 import java.util.Iterator;
 
 class QueryResultImpl extends AbstractIterator implements QueryResult {
 
-  private static final ImmutableMap
-      RESULT_TYPE_CONVERTER;
-
   private final DatastoreServiceImpl datastore;
   private final DatastoreV1.ReadOptions readOptionsPb;
   private final DatastoreV1.PartitionId partitionIdPb;
-  private final Query.ResultClass resultClass;
+  private final Query.Type queryType;
   private Query query;
-  private Query.Type type;
+  private Query.Type actualType;
   private DatastoreV1.QueryResultBatch resultPb;
   private Iterator entityResultPbIter;
   //private ByteString cursor; // only available in v1beta3
 
-  static {
-    ImmutableMap.Builder builder =
-        ImmutableMap.builder();
-    for (DatastoreV1.EntityResult.ResultType type : DatastoreV1.EntityResult.ResultType.values()) {
-      builder.put(type, Query.Type.valueOf(type.name()));
-    }
-    RESULT_TYPE_CONVERTER = builder.build();
-  }
 
   QueryResultImpl(DatastoreServiceImpl datastore, DatastoreV1.ReadOptions readOptionsPb,
       Query query) {
     this.datastore = datastore;
     this.readOptionsPb = readOptionsPb;
     this.query = query;
-    resultClass = query.resultClass();
+    queryType = query.type();
     DatastoreV1.PartitionId.Builder pbBuilder = DatastoreV1.PartitionId.newBuilder();
     pbBuilder.setDatasetId(datastore.options().dataset());
     if (query.namespace() != null) {
@@ -55,15 +44,20 @@ private DatastoreV1.QueryResultBatch sendRequest() {
     }
     requestPb.setPartitionId(partitionIdPb);
     query.populatePb(requestPb);
+    System.out.println("***************** KOKO ************************");
+    System.out.println("Request Query: " + requestPb.build());
     resultPb = datastore.run(requestPb.build()).getBatch();
+    System.out.println("Result: " + resultPb);
+    System.out.println("***************** KOKO ************************");
     entityResultPbIter = resultPb.getEntityResultList().iterator();
     // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
-    type = RESULT_TYPE_CONVERTER.get(resultPb.getEntityResultType());
-    Preconditions.checkState(resultClass.isAssignableFrom(type.resultClass()),
-        "Unexpected result type");
+    actualType = Type.fromPb(resultPb.getEntityResultType());
+    Preconditions.checkState(queryType.isAssignableFrom(actualType),
+        "Unexpected result type " + actualType + " vs " + queryType);
     return resultPb;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   protected T computeNext() {
     while (!entityResultPbIter.hasNext()
@@ -76,12 +70,12 @@ protected T computeNext() {
     }
     DatastoreV1.EntityResult entityResultPb = entityResultPbIter.next();
     //cursor = entityResultPb.getCursor(); // only available in v1beta3
-    return type.convert(entityResultPb.getEntity());
+    return (T) actualType.convert(entityResultPb.getEntity());
   }
 
   @Override
   public Class resultClass() {
-    return type.resultClass().value();
+    return actualType.resultClass();
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 328d8a896d8c..83a20f0efa24 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -41,7 +41,7 @@
  * 
A less trivial example of a projection query that returns the first 10 results
  * of "age" and "name" properties (sorted and grouped by "age") with an age greater than 18
  * 
 {@code
- *   StructuredQuery query = StructuredQuery.projectionBuilder()
+ *   StructuredQuery query = StructuredQuery.projectionBuilder()
  *       .kind(kind)
  *       .projection(Projection.property("age"), Projection.first("name"))
  *       .filter(PropertyFilter.gt("age", 18))
@@ -49,7 +49,7 @@
  *       .orderBy(OrderBy.asc("age"))
  *       .limit(10)
  *       .build();
- *   QueryResult results = datastore.run(query);
+ *   QueryResult results = datastore.run(query);
  *   ...
  * } 

  *
@@ -577,7 +577,7 @@ public static Projection first(String property) {
 
   static class BaseBuilder> {
 
-    private ResultClass resultClass;
+    private Type type;
     private String namespace;
     private String kind;
     private List projection = new LinkedList<>();
@@ -589,8 +589,8 @@ static class BaseBuilder> {
     private int offset;
     private Integer limit;
 
-    BaseBuilder(ResultClass resultClass) {
-      this.resultClass = resultClass;
+    BaseBuilder(Type type) {
+      this.type = type;
     }
 
     @SuppressWarnings("unchecked")
@@ -730,15 +730,15 @@ public StructuredQuery build() {
 
   public static final class Builder extends BaseBuilder> {
 
-    public Builder(ResultClass resultClass) {
-      super(resultClass);
+    public Builder(Type type) {
+      super(type);
     }
   }
 
   public static final class KeyOnlyBuilder extends BaseBuilder {
 
     public KeyOnlyBuilder() {
-      super(ResultClass.keyOnly());
+      super(Type.KEY_ONLY);
       projection(Projection.property(KEY_PROPERTY_NAME));
     }
 
@@ -757,10 +757,10 @@ public KeyOnlyQuery build() {
   }
 
   public static final class ProjectionBuilder
-      extends BaseBuilder {
+      extends BaseBuilder {
 
     public ProjectionBuilder() {
-      super(ResultClass.projection());
+      super(Type.PROJECTION);
     }
 
     @Override
@@ -808,7 +808,7 @@ public static final class KeyOnlyQuery extends StructuredQuery {
     }
   }
 
-  public static final class ProjectionQuery extends StructuredQuery {
+  public static final class ProjectionQuery extends StructuredQuery {
 
     private static final long serialVersionUID = -3333183044486150649L;
 
@@ -830,7 +830,7 @@ public List groupBy() {
   }
 
   StructuredQuery(BaseBuilder builder) {
-    super(builder.resultClass, builder.namespace);
+    super(builder.type, builder.namespace);
     kind = builder.kind;
     projection = ImmutableList.copyOf(builder.projection);
     filter = builder.filter;
@@ -917,7 +917,7 @@ protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
 
   @Override
   protected StructuredQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
-    Builder builder = new Builder<>(resultClass());
+    Builder builder = new Builder<>(type());
     builder.mergeFrom(toPb());
     builder.startCursor(new Cursor(responsePb.getEndCursor()));
     if (offset > 0 && responsePb.getSkippedResults() < offset) {
@@ -965,17 +965,16 @@ protected DatastoreV1.Query toPb() {
   }
 
   @Override
-  protected Object fromPb(ResultClass resultClass, String namespace, byte[] bytesPb)
+  protected Object fromPb(Type type, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
-    return fromPb(resultClass, namespace, DatastoreV1.Query.parseFrom(bytesPb));
+    return fromPb(type, namespace, DatastoreV1.Query.parseFrom(bytesPb));
   }
 
-  static StructuredQuery fromPb(ResultClass resultClass, String namespace,
-      DatastoreV1.Query queryPb) {
+  static StructuredQuery fromPb(Type type, String namespace, DatastoreV1.Query queryPb) {
     BaseBuilder builder;
-    if (resultClass.equals(ResultClass.full())) {
+    if (type.equals(Type.FULL)) {
       builder = builder();
-    } else if (resultClass.equals(ResultClass.keyOnly())) {
+    } else if (type.equals(Type.KEY_ONLY)) {
       builder = keyOnlyBuilder();
     } else {
       builder = projectionBuilder();
@@ -984,7 +983,7 @@ static StructuredQuery fromPb(ResultClass resultClass, String namespace,
   }
 
   public static Builder builder() {
-    return new Builder<>(ResultClass.full());
+    return new Builder<>(Type.FULL);
   }
 
   public static KeyOnlyBuilder keyOnlyBuilder() {
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index a2e15b311689..8b43f421b9e5 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -6,7 +6,9 @@
 import com.google.gcloud.datastore.TransactionOption.IsolationLevel;
 import com.google.protobuf.ByteString;
 
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 public final class TransactionImpl extends BatchWriterImpl implements Transaction {
@@ -15,11 +17,11 @@ public final class TransactionImpl extends BatchWriterImpl implements Transactio
   private boolean wasRolledback;
 
   TransactionImpl(DatastoreServiceImpl datastore, TransactionOption... options) {
-    super(datastore, options);
+    super(datastore, getBatchOptions(options));
     DatastoreV1.BeginTransactionRequest.Builder requestPb =
         DatastoreV1.BeginTransactionRequest.newBuilder();
-    Map, BatchWriteOption> optionsMap =
-        BatchWriteOption.asImmutableMap(options);
+    Map, TransactionOption> optionsMap =
+        TransactionOption.asImmutableMap(options);
     IsolationLevel isolationLevel = (IsolationLevel) optionsMap.get(IsolationLevel.class);
     if (isolationLevel != null) {
       requestPb.setIsolationLevel(isolationLevel.level().toPb());
@@ -27,6 +29,17 @@ public final class TransactionImpl extends BatchWriterImpl implements Transactio
     transaction = datastore.requestTransactionId(requestPb);
   }
 
+  private static BatchWriteOption[] getBatchOptions(TransactionOption... options) {
+    List batchOptions = new ArrayList<>(options.length);
+    for (TransactionOption option : options) {
+      BatchWriteOption batchOption = option.toBatchWriteOption();
+      if (batchOption != null) {
+        batchOptions.add(batchOption);
+      }
+    }
+    return batchOptions.toArray(new BatchWriteOption[batchOptions.size()]);
+  }
+
   @Override
   public Entity get(Key key) {
     Iterator iter = get(key, DatastoreServiceImpl.EMPTY_KEY_ARRAY);
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
index 0471929e7fb1..597715df20a8 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionOption.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
@@ -1,12 +1,36 @@
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.collect.ImmutableMap;
 
+import java.io.Serializable;
+import java.util.Map;
 
-public abstract class TransactionOption extends BatchWriteOption {
+
+public abstract class TransactionOption implements Serializable {
 
   private static final long serialVersionUID = -1862234444015690375L;
 
+  public static final class ForceWrites extends TransactionOption {
+
+    private static final long serialVersionUID = -6873967516988380886L;
+
+    private final boolean force;
+
+    public ForceWrites(boolean force) {
+      this.force = force;
+    }
+
+    public boolean force() {
+      return force;
+    }
+
+    @Override
+    BatchWriteOption toBatchWriteOption() {
+      return new BatchWriteOption.ForceWrites(force);
+    }
+  }
+
   public static final class IsolationLevel extends TransactionOption {
 
     private static final long serialVersionUID = -5592165378565409515L;
@@ -43,6 +67,10 @@ public Level level() {
     // package protected
   }
 
+  public static ForceWrites forceWrites() {
+    return new ForceWrites(true);
+  }
+
   public static IsolationLevel serializable() {
     return new IsolationLevel(IsolationLevel.Level.SERIALIZABLE);
   }
@@ -50,4 +78,18 @@ public static IsolationLevel serializable() {
   public static IsolationLevel snapshot() {
     return new IsolationLevel(IsolationLevel.Level.SNAPSHOT);
   }
+
+  static Map, TransactionOption> asImmutableMap(
+      TransactionOption... options) {
+    ImmutableMap.Builder, TransactionOption> builder =
+        ImmutableMap.builder();
+    for (TransactionOption option : options) {
+      builder.put(option.getClass(), option);
+    }
+    return builder.build();
+  }
+
+  BatchWriteOption toBatchWriteOption() {
+    return null;
+  }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index a7c6626d6b92..f3954cabbc23 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -9,7 +9,7 @@
 import static org.junit.Assert.fail;
 
 import com.google.common.collect.Maps;
-import com.google.gcloud.datastore.Query.ResultClass;
+import com.google.gcloud.datastore.Query.Type;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
@@ -45,13 +45,18 @@ public class DatastoreServiceTest {
       .addValue(STR_VALUE, BOOL_VALUE)
       .build();
   private static final ListValue LIST_VALUE2 = ListValue.of(Collections.singletonList(KEY_VALUE));
+  private static final DateTimeValue DATE_TIME_VALUE = new DateTimeValue(DateTime.now());
   private static final PartialEntity PARTIAL_ENTITY1 = PartialEntity.builder(PARTIAL_KEY2)
       .set("str", STR_VALUE).set("bool", BOOL_VALUE).set("list", LIST_VALUE1).build();
   private static final PartialEntity PARTIAL_ENTITY2 = PartialEntity.builder(PARTIAL_ENTITY1)
       .remove("str").set("bool", true).set("list", LIST_VALUE1.get()).build();
-  private static final Entity ENTITY1 = Entity.builder(KEY1).set("str", STR_VALUE)
-      .set("bool", BOOL_VALUE).set("partial1", EntityValue.of(PARTIAL_ENTITY1))
-      .set("list", LIST_VALUE2).build();
+  private static final Entity ENTITY1 = Entity.builder(KEY1)
+      .set("str", STR_VALUE)
+      .set("date", DATE_TIME_VALUE)
+      .set("bool", BOOL_VALUE)
+      .set("partial1", EntityValue.of(PARTIAL_ENTITY1))
+      .set("list", LIST_VALUE2)
+      .build();
   private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
       .set("name", "koko").setNull("null").set("age", 20).build();
   private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
@@ -272,7 +277,7 @@ public void testNewBatchWriter() {
 
   @Test
   public void testRunGqlQueryNoCasting() {
-    Query query1 = GqlQuery.builder(ResultClass.full(), "select * from " + KIND1).build();
+    Query query1 = GqlQuery.builder(Type.FULL, "select * from " + KIND1).build();
     QueryResult results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
@@ -280,7 +285,7 @@ public void testRunGqlQueryNoCasting() {
 
     datastore.put(ENTITY3);
     Query query2 =  GqlQuery.builder(
-        ResultClass.full(), "select * from " + KIND2 + " order by __key__").build();
+        Type.FULL, "select * from " + KIND2 + " order by __key__").build();
     QueryResult results2 = datastore.run(query2);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY2, results2.next());
@@ -288,24 +293,28 @@ public void testRunGqlQueryNoCasting() {
     assertEquals(ENTITY3, results2.next());
     assertFalse(results2.hasNext());
 
-    query1 = GqlQuery.builder(ResultClass.full(), "select * from bla").build();
+    query1 = GqlQuery.builder(Type.FULL, "select * from bla").build();
     results1 = datastore.run(query1);
     assertFalse(results1.hasNext());
 
     Query keyOnlyQuery =
-        GqlQuery.builder(ResultClass.keyOnly(), "select __key__ from " + KIND1).build();
+        GqlQuery.builder(Type.KEY_ONLY, "select __key__ from " + KIND1).build();
     QueryResult keyOnlyResults = datastore.run(keyOnlyQuery);
     assertTrue(keyOnlyResults.hasNext());
     assertEquals(KEY1, keyOnlyResults.next());
     assertFalse(keyOnlyResults.hasNext());
 
-    Query projectionQuery = GqlQuery.builder(
-        ResultClass.projection(), "select str from " + KIND1).build();
-    QueryResult projectionResult = datastore.run(projectionQuery);
+    // broken because of b/18806697
+    Query projectionQuery = GqlQuery.builder(
+        Type.PROJECTION, "select str, date from " + KIND1).build();
+    QueryResult projectionResult = datastore.run(projectionQuery);
     assertTrue(projectionResult.hasNext());
-    PartialEntity partialEntity = projectionResult.next();
-    assertEquals("str", partialEntity.getString("str"));
-    assertEquals(1, partialEntity.names().size());
+    ProjectionEntity projectionEntity = projectionResult.next();
+    assertEquals("str", projectionEntity.getString("str"));
+    assertEquals(DATE_TIME_VALUE, projectionEntity.getDateTime("date"));
+    assertEquals(DATE_TIME_VALUE.get().timestampMicroseconds(),
+        projectionEntity.getLong("date"));
+    assertEquals(2, projectionEntity.names().size());
     assertFalse(projectionResult.hasNext());
   }
 
@@ -343,7 +352,7 @@ public void testRunStructuredQuery() {
     assertEquals(ENTITY1.key(), results2.next());
     assertFalse(results2.hasNext());
 
-    StructuredQuery projectionQuery = StructuredQuery.projectionBuilder()
+    StructuredQuery projectionQuery = StructuredQuery.projectionBuilder()
         .kind(KIND2)
         .projection(Projection.property("age"), Projection.first("name"))
         .filter(PropertyFilter.gt("age", 18))
@@ -352,9 +361,10 @@ public void testRunStructuredQuery() {
         .limit(10)
         .build();
 
-    QueryResult results3 = datastore.run(projectionQuery);
+    // broken because of b/18806697
+    QueryResult results3 = datastore.run(projectionQuery);
     assertTrue(results3.hasNext());
-    PartialEntity entity = results3.next();
+    ProjectionEntity entity = results3.next();
     assertEquals(ENTITY2.key(), entity.key());
     assertEquals(20, entity.getLong("age"));
     assertEquals("koko", entity.getString("name"));
@@ -419,11 +429,14 @@ public void testGet() {
     StringValue value1 = entity.getValue("str");
     BooleanValue value2 = entity.getValue("bool");
     ListValue value3 = entity.getValue("list");
-    assertEquals(value1, STR_VALUE);
-    assertEquals(value2, BOOL_VALUE);
-    assertEquals(value3, LIST_VALUE2);
-    assertEquals(PARTIAL_ENTITY1, entity.getEntity("partial1"));
-    assertEquals(4, entity.names().size());
+    DateTimeValue value4 = entity.getValue("date");
+    PartialEntity value5 = entity.getEntity("partial1");
+    assertEquals(STR_VALUE, value1);
+    assertEquals(BOOL_VALUE, value2);
+    assertEquals(LIST_VALUE2, value3);
+    assertEquals(DATE_TIME_VALUE, value4);
+    assertEquals(PARTIAL_ENTITY1, value5);
+    assertEquals(5, entity.names().size());
     assertFalse(entity.contains("bla"));
   }
 
@@ -445,7 +458,7 @@ public void testGetArray() {
     assertEquals(partial1, PARTIAL_ENTITY2);
     assertEquals(partial2, ENTITY2);
     assertEquals(Value.Type.BOOLEAN, entity3.type("bool"));
-    assertEquals(5, entity3.names().size());
+    assertEquals(6, entity3.names().size());
     assertFalse(entity3.contains("bla"));
     try {
       entity3.getString("str");
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index f66bb6bc319a..27ec3b600d58 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -7,7 +7,6 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Multimap;
-import com.google.gcloud.datastore.Query.ResultClass;
 import com.google.gcloud.datastore.StructuredQuery.CompositeFilter;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
@@ -40,7 +39,7 @@ public class SerializationTest {
       .namespace("ns1")
       .build();
   private static final Query GQL2 =
-      GqlQuery.builder(ResultClass.full(), "select * from kind1 where name = @name and age > @1")
+      GqlQuery.builder(Query.Type.FULL, "select * from kind1 where name = @name and age > @1")
       .setArgument("name", "name1")
       .addArgument(20)
       .namespace("ns1")
@@ -50,7 +49,7 @@ public class SerializationTest {
       .kind("k")
       .filter(PropertyFilter.eq("p1", "hello"))
       .build();
-  private static final Query QUERY3 = StructuredQuery.projectionBuilder()
+  private static final Query QUERY3 = StructuredQuery.projectionBuilder()
       .kind("k")
       .namespace("ns1")
       .projection(Projection.property("p"))
@@ -95,6 +94,7 @@ public class SerializationTest {
       .addValue(STRING_VALUE)
       .addValue(new NullValue())
       .build();
+  private static final ProjectionEntity PROJECTION_ENTITY = ProjectionEntity.fromPb(ENTITY1.toPb());
 
   @SuppressWarnings("rawtypes")
   private Multimap typeToValues = ImmutableMultimap.builder()
@@ -129,8 +129,8 @@ public void testValues() throws Exception {
   @Test
   public void testTypes() throws Exception {
     Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
-        ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3, DATE_TIME1,
-        BLOB1, CURSOR1, GQL1, GQL2, QUERY1, QUERY2, QUERY3};
+        ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3, PROJECTION_ENTITY,
+        DATE_TIME1, BLOB1, CURSOR1, GQL1, GQL2, QUERY1, QUERY2, QUERY3};
     for (Object obj : types) {
       Object copy = serialiazeAndDeserialize(obj);
       assertEquals(obj, obj);

From 72a1192efae97ee0b0c05b9d18340094c612fa88 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 19 Dec 2014 09:37:10 -0800
Subject: [PATCH 065/732] Add a hack to the test to by-pass datastore issue
 with projection query

---
 pom.xml                                       |  6 +++
 .../datastore/DatastoreServiceFactory.java    |  7 +--
 .../datastore/DatastoreServiceOptions.java    | 16 +++++-
 .../datastore/DatastoreServiceTest.java       | 50 ++++++++++++++++---
 4 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/pom.xml b/pom.xml
index 1fa28d56c920..1f8cb504a65c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,6 +65,12 @@
       google-api-services-datastore
       v1beta2-rev23-1.19.0
     
+    
+    	org.easymock
+    	easymock
+    	3.3
+    	test
+    
   
   
     
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
index 2ae4a3011836..9a8bb1dd82bc 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
@@ -1,8 +1,5 @@
 package com.google.gcloud.datastore;
 
-import com.google.api.services.datastore.client.Datastore;
-import com.google.api.services.datastore.client.DatastoreFactory;
-import com.google.api.services.datastore.client.DatastoreOptions;
 
 
 public abstract class DatastoreServiceFactory {
@@ -10,9 +7,7 @@ public abstract class DatastoreServiceFactory {
   private static final DatastoreServiceFactory INSTANCE = new DatastoreServiceFactory() {
       @Override
       public DatastoreService get(DatastoreServiceOptions options) {
-        DatastoreOptions dsOptions = options.toDatastoreOptions();
-        Datastore datastore = DatastoreFactory.get().create(dsOptions);
-        return new DatastoreServiceImpl(options, datastore);
+        return new DatastoreServiceImpl(options, options.datastore());
       }
     };
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 4931bd62af71..3b81cac646c4 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -5,6 +5,8 @@
 import static com.google.gcloud.datastore.Validator.validateDataset;
 import static com.google.gcloud.datastore.Validator.validateNamespace;
 
+import com.google.api.services.datastore.client.Datastore;
+import com.google.api.services.datastore.client.DatastoreFactory;
 import com.google.api.services.datastore.client.DatastoreOptions;
 import com.google.common.collect.ImmutableSet;
 import com.google.gcloud.ServiceOptions;
@@ -20,6 +22,7 @@ public class DatastoreServiceOptions extends ServiceOptions {
   private final String dataset;
   private final String namespace;
   private final boolean force;
+  private final Datastore datastore;
 
   DatastoreServiceOptions(Builder builder) {
     super(builder);
@@ -27,6 +30,7 @@ public class DatastoreServiceOptions extends ServiceOptions {
     checkArgument(dataset != null, "missing dataset");
     namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
     force = builder.force;
+    datastore = builder.datastore;
   }
 
   public static class Builder extends ServiceOptions.Builder {
@@ -34,6 +38,7 @@ public static class Builder extends ServiceOptions.Builder {
     private String dataset;
     private String namespace;
     private boolean force = false;
+    private Datastore datastore;
 
     private Builder() {
     }
@@ -49,6 +54,11 @@ public DatastoreServiceOptions build() {
       return new DatastoreServiceOptions(this);
     }
 
+    Builder datastore(Datastore datastore) {
+      this.datastore = datastore;
+      return this;
+    }
+
     public Builder dataset(String dataset) {
       this.dataset = validateDataset(dataset);
       return this;
@@ -100,7 +110,7 @@ public Builder toBuilder() {
     return new Builder(this);
   }
 
-  DatastoreOptions toDatastoreOptions() {
+  private DatastoreOptions toDatastoreOptions() {
     return new DatastoreOptions.Builder()
         .dataset(dataset())
         .host(host())
@@ -108,6 +118,10 @@ DatastoreOptions toDatastoreOptions() {
         .build();
   }
 
+  Datastore datastore() {
+    return datastore != null ? datastore : DatastoreFactory.get().create(toDatastoreOptions());
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index f3954cabbc23..4465ead8abb6 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -8,12 +8,16 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.api.services.datastore.DatastoreV1;
+import com.google.api.services.datastore.client.Datastore;
+import com.google.api.services.datastore.client.DatastoreException;
 import com.google.common.collect.Maps;
 import com.google.gcloud.datastore.Query.Type;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
 
+import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -276,7 +280,7 @@ public void testNewBatchWriter() {
   }
 
   @Test
-  public void testRunGqlQueryNoCasting() {
+  public void testRunGqlQueryNoCasting() throws DatastoreException {
     Query query1 = GqlQuery.builder(Type.FULL, "select * from " + KIND1).build();
     QueryResult results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
@@ -304,18 +308,35 @@ public void testRunGqlQueryNoCasting() {
     assertEquals(KEY1, keyOnlyResults.next());
     assertFalse(keyOnlyResults.hasNext());
 
-    // broken because of b/18806697
-    Query projectionQuery = GqlQuery.builder(
+    GqlQuery projectionQuery = GqlQuery.builder(
         Type.PROJECTION, "select str, date from " + KIND1).build();
+
+    // this hack is needed because of b/18806697
+    DatastoreV1.RunQueryRequest.Builder requestPb = DatastoreV1.RunQueryRequest.newBuilder();
+    requestPb.setGqlQuery(projectionQuery.toPb());
+    requestPb.setPartitionId(DatastoreV1.PartitionId.newBuilder().setDatasetId(DATASET).build());
+    DatastoreV1.RunQueryResponse responsePb =
+        ((DatastoreServiceImpl) datastore).run(requestPb.build());
+    DatastoreV1.RunQueryResponse.Builder responsePbBuilder = responsePb.toBuilder();
+    responsePbBuilder.getBatchBuilder()
+        .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();
+    Datastore mockDatastore = EasyMock.createMock(Datastore.class);
+    EasyMock.expect(mockDatastore.runQuery(requestPb.build())).andReturn(responsePbBuilder.build());
+    EasyMock.replay(mockDatastore);
+    datastore = DatastoreServiceFactory.getDefault(
+        datastore.options().toBuilder().datastore(mockDatastore).build());
+    // end of hack
+
     QueryResult projectionResult = datastore.run(projectionQuery);
     assertTrue(projectionResult.hasNext());
     ProjectionEntity projectionEntity = projectionResult.next();
     assertEquals("str", projectionEntity.getString("str"));
-    assertEquals(DATE_TIME_VALUE, projectionEntity.getDateTime("date"));
+    assertEquals(DATE_TIME_VALUE.get(), projectionEntity.getDateTime("date"));
     assertEquals(DATE_TIME_VALUE.get().timestampMicroseconds(),
         projectionEntity.getLong("date"));
     assertEquals(2, projectionEntity.names().size());
     assertFalse(projectionResult.hasNext());
+    EasyMock.verify(mockDatastore);
   }
 
   @Test
@@ -338,7 +359,7 @@ public void testRunGqlQueryWithCasting() {
   }
 
   @Test
-  public void testRunStructuredQuery() {
+  public void testRunStructuredQuery() throws DatastoreException {
     StructuredQuery query =
         StructuredQuery.builder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
     QueryResult results1 = datastore.run(query);
@@ -361,7 +382,22 @@ public void testRunStructuredQuery() {
         .limit(10)
         .build();
 
-    // broken because of b/18806697
+    // this hack is needed because of b/18806697
+    DatastoreV1.RunQueryRequest.Builder requestPb = DatastoreV1.RunQueryRequest.newBuilder();
+    requestPb.setQuery(projectionQuery.toPb());
+    requestPb.setPartitionId(DatastoreV1.PartitionId.newBuilder().setDatasetId(DATASET).build());
+    DatastoreV1.RunQueryResponse responsePb =
+        ((DatastoreServiceImpl) datastore).run(requestPb.build());
+    DatastoreV1.RunQueryResponse.Builder responsePbBuilder = responsePb.toBuilder();
+    responsePbBuilder.getBatchBuilder()
+        .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();
+    Datastore mockDatastore = EasyMock.createMock(Datastore.class);
+    EasyMock.expect(mockDatastore.runQuery(requestPb.build())).andReturn(responsePbBuilder.build());
+    EasyMock.replay(mockDatastore);
+    datastore = DatastoreServiceFactory.getDefault(
+        datastore.options().toBuilder().datastore(mockDatastore).build());
+    // end of hack
+
     QueryResult results3 = datastore.run(projectionQuery);
     assertTrue(results3.hasNext());
     ProjectionEntity entity = results3.next();
@@ -370,6 +406,7 @@ public void testRunStructuredQuery() {
     assertEquals("koko", entity.getString("name"));
     assertEquals(2, entity.properties().size());
     assertFalse(results3.hasNext());
+    EasyMock.verify(mockDatastore);
 
     // TODO(ozarov): construct a test to verify nextQuery/pagination
   }
@@ -467,6 +504,7 @@ public void testGetArray() {
       // expected - no such property
     }
     assertFalse(result.hasNext());
+    // TODO(ozarov): construct a test to verify more results
   }
 
   public List fetch(Key key, Key... others) {

From ad977279e7d9c4da9c79d6855dd8b6d3a1141de2 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 19 Dec 2014 10:01:49 -0800
Subject: [PATCH 066/732] remove printout leftover

---
 .../java/com/google/gcloud/datastore/QueryResultImpl.java     | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index 808d9e8d7662..4d92ec80ed72 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -44,11 +44,7 @@ private DatastoreV1.QueryResultBatch sendRequest() {
     }
     requestPb.setPartitionId(partitionIdPb);
     query.populatePb(requestPb);
-    System.out.println("***************** KOKO ************************");
-    System.out.println("Request Query: " + requestPb.build());
     resultPb = datastore.run(requestPb.build()).getBatch();
-    System.out.println("Result: " + resultPb);
-    System.out.println("***************** KOKO ************************");
     entityResultPbIter = resultPb.getEntityResultList().iterator();
     // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
     actualType = Type.fromPb(resultPb.getEntityResultType());

From 8ad09d4334532375be8963a278582248fb92646a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 19 Dec 2014 10:52:41 -0800
Subject: [PATCH 067/732] 1. Renamed nameArg and numberArg to namedBinding and
 positionalBinding 2. Allow projection query to accept KEY_ONLY or FULL
 results.

---
 .../com/google/gcloud/datastore/GqlQuery.java | 146 +++++++++---------
 .../gcloud/datastore/QueryResultImpl.java     |   4 +
 .../gcloud/datastore/StructuredQuery.java     |   2 -
 .../datastore/DatastoreServiceTest.java       |  27 +++-
 .../gcloud/datastore/SerializationTest.java   |   8 +-
 5 files changed, 103 insertions(+), 84 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index c80cb4a77f14..8569bae8c674 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -58,10 +58,10 @@ public final class GqlQuery extends Query {
 
   private final transient String queryString;
   private final transient boolean allowLiteral;
-  private final transient ImmutableList nameArgs;
-  private final transient ImmutableList numberArgs;
+  private final transient ImmutableList namedBindings;
+  private final transient ImmutableList positionalBindings;
 
-  static final class Argument extends Serializable {
+  static final class Binding extends Serializable {
 
     private static final long serialVersionUID = 1976895435257636275L;
 
@@ -69,13 +69,13 @@ static final class Argument extends Serializable {
     private final transient Cursor cursor;
     private final transient Value value;
 
-    Argument(String name, Cursor cursor) {
+    Binding(String name, Cursor cursor) {
       this.name = name;
       this.cursor = checkNotNull(cursor);
       value = null;
     }
 
-    Argument(String name, Value value) {
+    Binding(String name, Value value) {
       this.name = name;
       this.value = checkNotNull(value);
       cursor = null;
@@ -99,10 +99,10 @@ public boolean equals(Object obj) {
       if (obj == this) {
         return true;
       }
-      if (!(obj instanceof Argument)) {
+      if (!(obj instanceof Binding)) {
         return false;
       }
-      Argument other = (Argument) obj;
+      Binding other = (Binding) obj;
       return Objects.equals(name, other.name)
         && Objects.equals(cursor, other.cursor)
         && Objects.equals(value, other.value);
@@ -128,12 +128,12 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
       return fromPb(DatastoreV1.GqlQueryArg.parseFrom(bytesPb));
     }
 
-    static Argument fromPb(DatastoreV1.GqlQueryArg argPb) {
+    static Binding fromPb(DatastoreV1.GqlQueryArg argPb) {
       String name = argPb.hasName() ? argPb.getName() : null;
       if (argPb.hasCursor()) {
-        return new Argument(name, new Cursor(argPb.getCursor()));
+        return new Binding(name, new Cursor(argPb.getCursor()));
       }
-      return new Argument(name, Value.fromPb(argPb.getValue()));
+      return new Binding(name, Value.fromPb(argPb.getValue()));
     }
   }
 
@@ -146,8 +146,8 @@ public static final class Builder {
     private String namespace;
     private String queryString;
     private boolean allowLiteral;
-    private Map nameArgs = new TreeMap<>();
-    private List numberArgs = new LinkedList<>();
+    private Map namedBindings = new TreeMap<>();
+    private List positionalBindings = new LinkedList<>();
 
     Builder(Type type, String query) {
       this.type = checkNotNull(type);
@@ -169,99 +169,99 @@ public Builder allowLiteral(boolean allowLiteral) {
       return this;
     }
 
-    public Builder clearArguments() {
-      nameArgs.clear();
-      numberArgs.clear();
+    public Builder clearBindings() {
+      namedBindings.clear();
+      positionalBindings.clear();
       return this;
     }
 
-    public Builder setArgument(String name, Cursor cursor) {
-      nameArgs.put(name, new Argument(name, cursor));
+    public Builder setBinding(String name, Cursor cursor) {
+      namedBindings.put(name, new Binding(name, cursor));
       return this;
     }
 
-    public Builder setArgument(String name, String... value) {
-      nameArgs.put(name, toArgument(name, StringValue.MARSHALLER, Arrays.asList(value)));
+    public Builder setBinding(String name, String... value) {
+      namedBindings.put(name, toBinding(name, StringValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, long... value) {
-      nameArgs.put(name, toArgument(name, LongValue.MARSHALLER, Longs.asList(value)));
+    public Builder setBinding(String name, long... value) {
+      namedBindings.put(name, toBinding(name, LongValue.MARSHALLER, Longs.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, double... value) {
-      nameArgs.put(name, toArgument(name, DoubleValue.MARSHALLER, Doubles.asList(value)));
+    public Builder setBinding(String name, double... value) {
+      namedBindings.put(name, toBinding(name, DoubleValue.MARSHALLER, Doubles.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, boolean... value) {
-      nameArgs.put(name, toArgument(name, BooleanValue.MARSHALLER, Booleans.asList(value)));
+    public Builder setBinding(String name, boolean... value) {
+      namedBindings.put(name, toBinding(name, BooleanValue.MARSHALLER, Booleans.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, DateTime... value) {
-      nameArgs.put(name, toArgument(name, DateTimeValue.MARSHALLER, Arrays.asList(value)));
+    public Builder setBinding(String name, DateTime... value) {
+      namedBindings.put(name, toBinding(name, DateTimeValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, Key... value) {
-      nameArgs.put(name, toArgument(name, KeyValue.MARSHALLER, Arrays.asList(value)));
+    public Builder setBinding(String name, Key... value) {
+      namedBindings.put(name, toBinding(name, KeyValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, PartialEntity... value) {
-      nameArgs.put(name, toArgument(name, EntityValue.MARSHALLER, Arrays.asList(value)));
+    public Builder setBinding(String name, PartialEntity... value) {
+      namedBindings.put(name, toBinding(name, EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder setArgument(String name, Blob... value) {
-      nameArgs.put(name, toArgument(name, BlobValue.MARSHALLER, Arrays.asList(value)));
+    public Builder setBinding(String name, Blob... value) {
+      namedBindings.put(name, toBinding(name, BlobValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(Cursor cursor) {
-      numberArgs.add(new Argument(null, cursor));
+    public Builder addBinding(Cursor cursor) {
+      positionalBindings.add(new Binding(null, cursor));
       return this;
     }
 
-    public Builder addArgument(String... value) {
-      numberArgs.add(toArgument(StringValue.MARSHALLER, Arrays.asList(value)));
+    public Builder addBinding(String... value) {
+      positionalBindings.add(toBinding(StringValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(long... value) {
-      numberArgs.add(toArgument(LongValue.MARSHALLER, Longs.asList(value)));
+    public Builder addBinding(long... value) {
+      positionalBindings.add(toBinding(LongValue.MARSHALLER, Longs.asList(value)));
       return this;
     }
 
-    public Builder addArgument(double... value) {
-      numberArgs.add(toArgument(DoubleValue.MARSHALLER, Doubles.asList(value)));
+    public Builder addBinding(double... value) {
+      positionalBindings.add(toBinding(DoubleValue.MARSHALLER, Doubles.asList(value)));
       return this;
     }
 
-    public Builder addArgument(boolean... value) {
-      numberArgs.add(toArgument(BooleanValue.MARSHALLER, Booleans.asList(value)));
+    public Builder addBinding(boolean... value) {
+      positionalBindings.add(toBinding(BooleanValue.MARSHALLER, Booleans.asList(value)));
       return this;
     }
 
-    public Builder addArgument(DateTime... value) {
-      numberArgs.add(toArgument(DateTimeValue.MARSHALLER, Arrays.asList(value)));
+    public Builder addBinding(DateTime... value) {
+      positionalBindings.add(toBinding(DateTimeValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(Key... value) {
-      numberArgs.add(toArgument(KeyValue.MARSHALLER, Arrays.asList(value)));
+    public Builder addBinding(Key... value) {
+      positionalBindings.add(toBinding(KeyValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(PartialEntity... value) {
-      numberArgs.add(toArgument(EntityValue.MARSHALLER, Arrays.asList(value)));
+    public Builder addBinding(PartialEntity... value) {
+      positionalBindings.add(toBinding(EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
-    public Builder addArgument(Blob... value) {
-      numberArgs.add(toArgument(BlobValue.MARSHALLER, Arrays.asList(value)));
+    public Builder addBinding(Blob... value) {
+      positionalBindings.add(toBinding(BlobValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
 
@@ -270,12 +270,12 @@ public GqlQuery build() {
     }
 
     @SuppressWarnings("rawtypes")
-    private static Argument toArgument(Value.BuilderFactory builderFactory, List values) {
-      return toArgument(null, builderFactory, values);
+    private static Binding toBinding(Value.BuilderFactory builderFactory, List values) {
+      return toBinding(null, builderFactory, values);
     }
 
     @SuppressWarnings({"unchecked", "rawtypes"})
-    private static Argument toArgument(String name, Value.BuilderFactory builderFactory,
+    private static Binding toBinding(String name, Value.BuilderFactory builderFactory,
         List values) {
       List> list = new ArrayList<>(values.size());
       for (Object object : values) {
@@ -289,7 +289,7 @@ private static Argument toArgument(String name, Value.BuilderFactory builderFact
       } else {
         value = new ListValue(list);
       }
-      return new Argument(name, value);
+      return new Binding(name, value);
     }
   }
 
@@ -297,8 +297,8 @@ private GqlQuery(Builder builder) {
     super(builder.type, builder.namespace);
     queryString = builder.queryString;
     allowLiteral = builder.allowLiteral;
-    nameArgs = ImmutableList.copyOf(builder.nameArgs.values());
-    numberArgs = ImmutableList.copyOf(builder.numberArgs);
+    namedBindings = ImmutableList.copyOf(builder.namedBindings.values());
+    positionalBindings = ImmutableList.copyOf(builder.positionalBindings);
   }
 
   public String queryString() {
@@ -310,30 +310,30 @@ public boolean allowLiteral() {
   }
 
   /**
-   * Returns an immutable map of named arguments.
+   * Returns an immutable map of named bindings.
    */
-  public Map nameArgs() {
+  public Map namedBindings() {
     ImmutableMap.Builder builder = ImmutableSortedMap.naturalOrder();
-    for (Argument argument : nameArgs) {
-      builder.put(argument.name(), argument.cursorOrValue());
+    for (Binding binding : namedBindings) {
+      builder.put(binding.name(), binding.cursorOrValue());
     }
     return builder.build();
   }
 
   /**
-   * Returns an immutable list of numbered arguments (using original order).
+   * Returns an immutable list of positional bindings (using original order).
    */
   public List numberArgs() {
     ImmutableList.Builder builder = ImmutableList.builder();
-    for (Argument argument : numberArgs) {
-      builder.add(argument.cursorOrValue());
+    for (Binding binding : positionalBindings) {
+      builder.add(binding.cursorOrValue());
     }
     return builder.build();
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(namespace(), queryString, allowLiteral, nameArgs, numberArgs);
+    return Objects.hash(namespace(), queryString, allowLiteral, namedBindings, positionalBindings);
   }
 
   @Override
@@ -348,8 +348,8 @@ public boolean equals(Object obj) {
     return Objects.equals(namespace(), other.namespace())
         && Objects.equals(queryString, other.queryString)
         && allowLiteral == other.allowLiteral
-        && Objects.equals(nameArgs,  other.nameArgs)
-        && Objects.equals(numberArgs,  other.numberArgs);
+        && Objects.equals(namedBindings,  other.namedBindings)
+        && Objects.equals(positionalBindings,  other.positionalBindings);
   }
 
   @Override
@@ -357,10 +357,10 @@ protected DatastoreV1.GqlQuery toPb() {
     DatastoreV1.GqlQuery.Builder queryPb = DatastoreV1.GqlQuery.newBuilder();
     queryPb.setQueryString(queryString);
     queryPb.setAllowLiteral(allowLiteral);
-    for (Argument argument : nameArgs) {
+    for (Binding argument : namedBindings) {
       queryPb.addNameArg(argument.toPb());
     }
-    for (Argument argument : numberArgs) {
+    for (Binding argument : positionalBindings) {
       queryPb.addNumberArg(argument.toPb());
     }
     return queryPb.build();
@@ -391,12 +391,12 @@ static  GqlQuery fromPb(Type resultType, String namespace,
       builder.allowLiteral = queryPb.getAllowLiteral();
     }
     for (DatastoreV1.GqlQueryArg nameArg : queryPb.getNameArgList()) {
-      Argument argument = Argument.fromPb(nameArg);
-      builder.nameArgs.put(argument.name(), argument);
+      Binding argument = Binding.fromPb(nameArg);
+      builder.namedBindings.put(argument.name(), argument);
     }
     for (DatastoreV1.GqlQueryArg numberArg : queryPb.getNumberArgList()) {
-      Argument argument = Argument.fromPb(numberArg);
-      builder.numberArgs.add(argument);
+      Binding argument = Binding.fromPb(numberArg);
+      builder.positionalBindings.add(argument);
     }
     return builder.build();
   }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index 4d92ec80ed72..5639eec89195 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -48,6 +48,10 @@ private DatastoreV1.QueryResultBatch sendRequest() {
     entityResultPbIter = resultPb.getEntityResultList().iterator();
     // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
     actualType = Type.fromPb(resultPb.getEntityResultType());
+    if (queryType == Type.PROJECTION) {
+      // projection entity can represent all type of results
+      actualType = Type.PROJECTION;
+    }
     Preconditions.checkState(queryType.isAssignableFrom(actualType),
         "Unexpected result type " + actualType + " vs " + queryType);
     return resultPb;
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 83a20f0efa24..84d7d65e2c25 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -814,8 +814,6 @@ public static final class ProjectionQuery extends StructuredQuery keyProjectionQuery = GqlQuery.builder(
+        Type.PROJECTION, "select __key__ from " + KIND1).build();
+    QueryResult keyProjectionResult = datastore.run(keyProjectionQuery);
+    assertTrue(keyProjectionResult.hasNext());
+    ProjectionEntity p = keyProjectionResult.next();
+    assertEquals(KEY1, p.key());
+    assertTrue(p.properties().isEmpty());
+    assertFalse(keyProjectionResult.hasNext());
+
     GqlQuery projectionQuery = GqlQuery.builder(
         Type.PROJECTION, "select str, date from " + KIND1).build();
 
@@ -373,6 +382,15 @@ public void testRunStructuredQuery() throws DatastoreException {
     assertEquals(ENTITY1.key(), results2.next());
     assertFalse(results2.hasNext());
 
+    StructuredQuery keyOnlyProjectionQuery = StructuredQuery.projectionBuilder()
+        .kind(KIND1).projection(Projection.property("__key__")).build();
+    QueryResult results3 = datastore.run(keyOnlyProjectionQuery);
+    assertTrue(results3.hasNext());
+    ProjectionEntity projectionEntity = results3.next();
+    assertEquals(ENTITY1.key(), projectionEntity.key());
+    assertTrue(projectionEntity.names().isEmpty());
+    assertFalse(results2.hasNext());
+
     StructuredQuery projectionQuery = StructuredQuery.projectionBuilder()
         .kind(KIND2)
         .projection(Projection.property("age"), Projection.first("name"))
@@ -381,7 +399,6 @@ public void testRunStructuredQuery() throws DatastoreException {
         .orderBy(OrderBy.asc("age"))
         .limit(10)
         .build();
-
     // this hack is needed because of b/18806697
     DatastoreV1.RunQueryRequest.Builder requestPb = DatastoreV1.RunQueryRequest.newBuilder();
     requestPb.setQuery(projectionQuery.toPb());
@@ -398,14 +415,14 @@ public void testRunStructuredQuery() throws DatastoreException {
         datastore.options().toBuilder().datastore(mockDatastore).build());
     // end of hack
 
-    QueryResult results3 = datastore.run(projectionQuery);
-    assertTrue(results3.hasNext());
-    ProjectionEntity entity = results3.next();
+    QueryResult results4 = datastore.run(projectionQuery);
+    assertTrue(results4.hasNext());
+    ProjectionEntity entity = results4.next();
     assertEquals(ENTITY2.key(), entity.key());
     assertEquals(20, entity.getLong("age"));
     assertEquals("koko", entity.getString("name"));
     assertEquals(2, entity.properties().size());
-    assertFalse(results3.hasNext());
+    assertFalse(results4.hasNext());
     EasyMock.verify(mockDatastore);
 
     // TODO(ozarov): construct a test to verify nextQuery/pagination
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 27ec3b600d58..706c247c6623 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -34,14 +34,14 @@ public class SerializationTest {
   private static final Cursor CURSOR2 = Cursor.copyFrom(new byte[] {10});
   private static final Query GQL1 =
       GqlQuery.builder("select * from kind1 where name = @name and age > @1")
-      .setArgument("name", "name1")
-      .addArgument(20)
+      .setBinding("name", "name1")
+      .addBinding(20)
       .namespace("ns1")
       .build();
   private static final Query GQL2 =
       GqlQuery.builder(Query.Type.FULL, "select * from kind1 where name = @name and age > @1")
-      .setArgument("name", "name1")
-      .addArgument(20)
+      .setBinding("name", "name1")
+      .addBinding(20)
       .namespace("ns1")
       .build();
   private static final Query QUERY1 = StructuredQuery.builder().kind("kind1").build();

From 2105cb7514508a68a2d9819b721ec8c5bd78029a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 19 Dec 2014 15:49:54 -0800
Subject: [PATCH 068/732] Change key API, to make it clear that kind (for
 partial) or (kind, name) (kind, id) are part of the path and remove
 addAncestor from builder as setter only make the path order more clear.

---
 .../google/gcloud/datastore/BaseEntity.java   | 16 ----
 .../com/google/gcloud/datastore/BaseKey.java  | 74 ++++++++-----------
 .../java/com/google/gcloud/datastore/Key.java | 45 +++++------
 .../google/gcloud/datastore/KeyFactory.java   | 24 +++---
 .../google/gcloud/datastore/PartialKey.java   | 49 +++++-------
 .../google/gcloud/datastore/PathElement.java  | 41 ++++++----
 .../google/gcloud/ExceptionHandlerTest.java   |  1 -
 .../datastore/DatastoreServiceTest.java       | 21 +++---
 .../gcloud/datastore/SerializationTest.java   |  4 +-
 9 files changed, 120 insertions(+), 155 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 0ec36a2f0211..91665fd59987 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -13,7 +13,6 @@
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableSortedMap;
-import com.google.gcloud.datastore.Value.Type;
 
 import java.util.HashMap;
 import java.util.List;
@@ -150,10 +149,6 @@ public > V getValue(String name) {
     return property;
   }
 
-  public Type type(String name) {
-    return getValue(name).type();
-  }
-
   public boolean isNull(String name) {
     return getValue(name) instanceof NullValue;
   }
@@ -195,17 +190,6 @@ public Blob getBlob(String name) {
     return ((BlobValue) getValue(name)).get();
   }
 
-  /**
-   * Returns the property's value as a {@link RawValue}.
-   */
-  public RawValue asRawValue(String name) {
-    Value value = getValue(name);
-    if (value instanceof RawValue) {
-      return (RawValue) value;
-    }
-    return new RawValue(value.toPb());
-  }
-
   /**
    * Returns the properties name.
    */
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index d8a7a553c157..f9d961034941 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -1,13 +1,11 @@
 package com.google.gcloud.datastore;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.gcloud.datastore.Validator.validateDataset;
 import static com.google.gcloud.datastore.Validator.validateKind;
 import static com.google.gcloud.datastore.Validator.validateNamespace;
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 
 import java.util.LinkedList;
@@ -23,8 +21,7 @@ abstract class BaseKey extends Serializable {
 
   private final transient String dataset;
   private final transient String namespace;
-  private final transient ImmutableList ancestors;
-  private final transient String kind;
+  private final transient ImmutableList path;
 
   abstract static class Builder> {
 
@@ -37,15 +34,15 @@ abstract static class Builder> {
 
     public Builder(String dataset, String kind) {
       this.dataset = validateDataset(dataset);
-      this.kind = validateKind(kind);
       ancestors = new LinkedList<>();
+      this.kind = validateKind(kind);
     }
 
     public Builder(BaseKey copyFrom) {
       dataset = copyFrom.dataset();
       namespace = copyFrom.namespace();
-      kind = copyFrom.kind();
       ancestors = new LinkedList<>(copyFrom.ancestors());
+      kind = copyFrom.kind();
     }
 
     @SuppressWarnings("unchecked")
@@ -53,30 +50,21 @@ protected B self() {
       return (B) this;
     }
 
-    public B addAncestor(String kind, long id) {
-      checkArgument(id != 0, "id must not be equal to zero");
-      return addAncestor(new PathElement(kind, id));
+    public B ancestors(PathElement ancestor) {
+      Preconditions.checkState(ancestors.size() < MAX_PATH, "path can have at most 100 elements");
+      ancestors.add(ancestor);
+      return self();
     }
 
-    public B addAncestor(String kind, String name) {
-      checkArgument(Strings.isNullOrEmpty(name) , "name must not be empty or null");
-      checkArgument(name.length() <= 500, "name must not exceed 500 characters");
-      return addAncestor(new PathElement(kind, name));
+    public B ancestors(PathElement ancestor, PathElement... other) {
+      return ancestors(ImmutableList.builder().add(ancestor).add(other).build());
     }
 
-    public B addAncestor(PathElement... ancestor) {
-      Preconditions.checkState(ancestors.size() + ancestor.length <= MAX_PATH,
+    public B ancestors(Iterable ancestors) {
+      ImmutableList list = ImmutableList.copyOf(ancestors);
+      Preconditions.checkState(this.ancestors.size() + list.size() < MAX_PATH,
           "path can have at most 100 elements");
-      for (PathElement pathElement : ancestor) {
-        ancestors.add(pathElement);
-      }
-      return self();
-    }
-
-    public B addAncestors(Iterable ancestors) {
-      for (PathElement pathElement : ancestors) {
-        addAncestor(pathElement);
-      }
+      this.ancestors.addAll(list);
       return self();
     }
 
@@ -85,11 +73,6 @@ public B kind(String kind) {
       return self();
     }
 
-    public B clearPath() {
-      ancestors.clear();
-      return self();
-    }
-
     public B dataset(String dataset) {
       this.dataset = validateDataset(dataset);
       return self();
@@ -103,11 +86,10 @@ public B namespace(String namespace) {
     protected abstract BaseKey build();
   }
 
-  BaseKey(String dataset, String namespace, ImmutableList ancestors, String kind) {
+  BaseKey(String dataset, String namespace, ImmutableList path) {
     this.dataset = dataset;
     this.namespace = namespace;
-    this.ancestors = ancestors;
-    this.kind = kind;
+    this.path = path;
   }
 
   /**
@@ -128,19 +110,30 @@ public String namespace() {
    * Returns an immutable list with the key's ancestors.
    */
   public List ancestors() {
-    return ancestors;
+    return path().subList(0, path().size() - 1);
+  }
+
+  /**
+   * Returns an immutable list of the key's path (ancestors + self).
+   */
+  public List path() {
+    return path;
+  }
+
+  protected PathElement leaf() {
+    return path().get(path().size() - 1);
   }
 
   /**
    * Returns the key's kind.
    */
   public String kind() {
-    return kind;
+    return leaf().kind();
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(dataset(), namespace(), ancestors(), leaf());
+    return Objects.hash(dataset(), namespace(), path());
   }
 
   @Override
@@ -154,8 +147,7 @@ public boolean equals(Object obj) {
     PartialKey other = (PartialKey) obj;
     return Objects.equals(dataset(), other.dataset())
         && Objects.equals(namespace(), other.namespace())
-        && Objects.equals(ancestors(), other.ancestors())
-        && Objects.equals(leaf(), other.leaf());
+        && Objects.equals(path(), other.path());
   }
 
   @Override
@@ -171,13 +163,9 @@ protected DatastoreV1.Key toPb() {
     if (partitionIdPb.hasDatasetId() || partitionIdPb.hasNamespace()) {
       keyPb.setPartitionId(partitionIdPb.build());
     }
-    for (PathElement pathEntry : ancestors) {
+    for (PathElement pathEntry : path) {
       keyPb.addPathElement(pathEntry.toPb());
     }
-    PathElement leaf = leaf();
-    keyPb.addPathElement(leaf.toPb());
     return keyPb.build();
   }
-
-  protected abstract PathElement leaf();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index a9b7ae6a0965..113802a8ba5b 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -23,8 +23,6 @@ public final class Key extends PartialKey {
 
   private static final long serialVersionUID = 3160994559785491356L;
 
-  private final transient PathElement leaf;
-
   public static final class Builder extends BaseKey.Builder {
 
     private String name;
@@ -63,52 +61,50 @@ public Builder id(long id) {
 
     @Override
     public Key build() {
+      ImmutableList.Builder pathBuilder =
+          ImmutableList.builder().addAll(ancestors);
       if (id == null) {
-        return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, name);
+        pathBuilder.add(PathElement.of(kind, name));
+      } else {
+        pathBuilder.add(PathElement.of(kind, id));
       }
-      return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, id);
+      return new Key(dataset, namespace, pathBuilder.build());
     }
   }
 
-  Key(String dataset, String namespace,  ImmutableList ancestors,
-      String kind, String name) {
-    super(dataset, namespace, ancestors, kind);
-    leaf = new PathElement(kind, name);
-  }
-
-  Key(String dataset, String namespace,  ImmutableList ancestors,
-      String kind, long id) {
-    super(dataset, namespace, ancestors, kind);
-    leaf = new PathElement(kind, id);
+  Key(String dataset, String namespace, ImmutableList path) {
+    super(dataset, namespace, path);
+    Preconditions.checkArgument(nameOrId() != null);
   }
 
   public boolean hasId() {
-    return leaf.hasId();
+    return leaf().hasId();
   }
 
   /**
    * Returns the key's id or {@code null} if it has a name instead.
    */
   public Long id() {
-    return leaf.id();
+    return leaf().id();
   }
 
   public boolean hasName() {
-    return leaf.hasName();
+    return leaf().hasName();
   }
 
   /**
    * Returns the key's name or {@code null} if it has an id instead.
    */
   public String name() {
-    return leaf.name();
+    return leaf().name();
   }
 
   /**
    * Returns the key's id (as {@link Long}) or name (as {@link String}).
+   * Never {@code null}.
    */
   public Object nameOrId() {
-    return leaf.nameOrId();
+    return leaf().nameOrId();
   }
 
   /**
@@ -139,11 +135,6 @@ public static Key fromUrlSafe(String urlSafe) {
     }
   }
 
-  @Override
-  protected PathElement leaf() {
-    return leaf;
-  }
-
   @Override
   protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(DatastoreV1.Key.parseFrom(bytesPb));
@@ -181,11 +172,11 @@ public static Builder builder(Key parent, String kind, long id) {
 
   private static void addParentToBuilder(Key parent, Builder builder) {
     builder.namespace(parent.namespace());
-    builder.addAncestors(parent.ancestors());
+    builder.ancestors(parent.ancestors());
     if (parent.hasId()) {
-      builder.addAncestor(new PathElement(parent.kind(), parent.id()));
+      builder.ancestors(PathElement.of(parent.kind(), parent.id()));
     } else {
-      builder.addAncestor(new PathElement(parent.kind(), parent.name()));
+      builder.ancestors(PathElement.of(parent.kind(), parent.name()));
     }
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index f671e0f72da5..4984d71100fe 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -18,21 +18,22 @@ public KeyFactory(DatastoreService service, String kind) {
     namespace(service.options().namespace());
   }
 
-  @Override
-  protected PartialKey build() {
-    return new PartialKey(dataset, namespace, ImmutableList.copyOf(ancestors), kind);
-  }
-
   public PartialKey newKey() {
-    return build();
+    ImmutableList path = ImmutableList.builder()
+        .addAll(ancestors).add(PathElement.of(kind)).build();
+    return new PartialKey(dataset, namespace, path);
   }
 
   public Key newKey(String name) {
-    return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, name);
+    ImmutableList path = ImmutableList.builder()
+        .addAll(ancestors).add(PathElement.of(kind, name)).build();
+    return new Key(dataset, namespace, path);
   }
 
   public Key newKey(long id) {
-    return new Key(dataset, namespace, ImmutableList.copyOf(ancestors), kind, id);
+    ImmutableList path = ImmutableList.builder()
+        .addAll(ancestors).add(PathElement.of(kind, id)).build();
+    return new Key(dataset, namespace, path);
   }
 
   /**
@@ -40,6 +41,11 @@ public Key newKey(long id) {
    * @throws DatastoreServiceException if allocation failed.
    */
   public Key allocateId() {
-    return service.allocateId(build());
+    return service.allocateId(newKey());
+  }
+
+  @Override
+  protected PartialKey build() {
+    return newKey();
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java
index 7630f3ebf275..302abbd065c8 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java
@@ -1,5 +1,6 @@
 package com.google.gcloud.datastore;
 
+import com.google.api.client.repackaged.com.google.common.base.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.InvalidProtocolBufferException;
@@ -27,25 +28,23 @@ private Builder(PartialKey copyFrom) {
 
     @Override
     public PartialKey build() {
-      return new PartialKey(dataset, namespace, ImmutableList.copyOf(ancestors), kind);
+      ImmutableList path = ImmutableList.builder()
+          .addAll(ancestors).add(PathElement.of(kind)).build();
+      return new PartialKey(dataset, namespace, path);
     }
   }
 
-  PartialKey(String dataset, String namespace, ImmutableList ancestors, String kind) {
-    super(dataset, namespace, ancestors, kind);
+  PartialKey(String dataset, String namespace, ImmutableList path) {
+    super(dataset, namespace, path);
   }
 
   public Key newKey(String name) {
-    return new Key(dataset(), namespace(), ImmutableList.copyOf(ancestors()), kind(), name);
+    return Key.builder(dataset(), kind(), name)
+        .namespace(namespace()).ancestors(ancestors()).build();
   }
 
   public Key newKey(long id) {
-    return new Key(dataset(), namespace(), ImmutableList.copyOf(ancestors()), kind(), id);
-  }
-
-  @Override
-  protected PathElement leaf() {
-    return new PathElement(kind());
+    return Key.builder(dataset(), kind(), id).namespace(namespace()).ancestors(ancestors()).build();
   }
 
   @Override
@@ -66,21 +65,17 @@ static PartialKey fromPb(DatastoreV1.Key keyPb) {
       }
     }
     List pathElementsPb = keyPb.getPathElementList();
-    if (pathElementsPb.isEmpty()) {
-      return new PartialKey(dataset, namespace, ImmutableList.of(), null);
-    }
+    Preconditions.checkArgument(pathElementsPb.size() > 0, "Path must not be empty");
     ImmutableList.Builder pathBuilder = ImmutableList.builder();
-    for (int i = 0; i < pathElementsPb.size() - 1; i++) {
-      pathBuilder.add(PathElement.fromPb(pathElementsPb.get(i)));
+    for (DatastoreV1.Key.PathElement pathElementPb : pathElementsPb) {
+      pathBuilder.add(PathElement.fromPb(pathElementPb));
     }
-    DatastoreV1.Key.PathElement leaf = pathElementsPb.get(pathElementsPb.size() - 1);
-    String kind = leaf.getKind();
-    if (leaf.hasId()) {
-      return new Key(dataset, namespace, pathBuilder.build(), kind, leaf.getId());
-    } else if (leaf.hasName()) {
-      return new Key(dataset, namespace, pathBuilder.build(), kind, leaf.getName());
+    ImmutableList path = pathBuilder.build();
+    PathElement leaf = path.get(path.size() - 1);
+    if (leaf.nameOrId() != null) {
+      return new Key(dataset, namespace, path);
     }
-    return new PartialKey(dataset, namespace, pathBuilder.build(), kind);
+    return new PartialKey(dataset, namespace, path);
   }
 
   public static Builder builder(String dataset, String kind) {
@@ -92,14 +87,6 @@ public static Builder builder(PartialKey copyFrom) {
   }
 
   public static Builder builder(Key parent, String kind) {
-    Builder builder = new Builder(parent.dataset(), kind)
-        .namespace(parent.namespace())
-        .addAncestors(parent.ancestors());
-    if (parent.hasId()) {
-      builder.addAncestor(new PathElement(parent.kind(), parent.id()));
-    } else {
-      builder.addAncestor(new PathElement(parent.kind(), parent.name()));
-    }
-    return builder;
+    return builder(parent.dataset(), kind).namespace(parent.namespace()).ancestors(parent.path());
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/PathElement.java b/src/main/java/com/google/gcloud/datastore/PathElement.java
index 34dcc06de4e6..82a4545faf8e 100644
--- a/src/main/java/com/google/gcloud/datastore/PathElement.java
+++ b/src/main/java/com/google/gcloud/datastore/PathElement.java
@@ -1,6 +1,10 @@
 package com.google.gcloud.datastore;
 
+import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.base.Strings;
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.util.Objects;
@@ -16,20 +20,10 @@ public final class PathElement extends Serializable
   private final transient Long id;
   private final transient String name;
 
-  PathElement(String kind) {
-    this(kind, null);
-  }
-
-  public PathElement(String kind, long id) {
-    this.kind = kind;
-    this.id = id;
-    name = null;
-  }
-
-  public PathElement(String kind, String name) {
-    this.kind = kind;
+  private PathElement(String kind, String name, Long id) {
+    this.kind = checkNotNull(kind);
     this.name = name;
-    id = null;
+    this.id = id;
   }
 
   public String kind() {
@@ -95,10 +89,25 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   static PathElement fromPb(DatastoreV1.Key.PathElement pathElementPb) {
     String kind = pathElementPb.getKind();
     if (pathElementPb.hasId()) {
-      return new PathElement(kind, pathElementPb.getId());
+      return PathElement.of(kind, pathElementPb.getId());
     } else if (pathElementPb.hasName()) {
-      return new PathElement(kind, pathElementPb.getName());
+      return PathElement.of(kind, pathElementPb.getName());
     }
-    return new PathElement(kind);
+    return PathElement.of(kind);
+  }
+
+  static PathElement of(String kind) {
+    return new PathElement(kind, null, null);
+  }
+
+  public static PathElement of(String kind, String name) {
+    checkArgument(!Strings.isNullOrEmpty(name) , "name must not be empty or null");
+    checkArgument(name.length() <= 500, "name must not exceed 500 characters");
+    return new PathElement(kind, name, null);
+  }
+
+  public static PathElement of(String kind, long id) {
+    checkArgument(id != 0, "id must not be equal to zero");
+    return new PathElement(kind, null, id);
   }
 }
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
index 10bd594b46a2..42905c85a08a 100644
--- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -12,7 +12,6 @@
 import java.io.IOException;
 import java.nio.channels.ClosedByInterruptException;
 import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 864d60f6160b..a1d021c3739d 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -241,6 +241,7 @@ public void testNewBatchWriter() {
     Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
     Entity entity4 = Entity.builder(KEY4).set("value", StringValue.of("value")).build();
     Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
+
     batchWriter.add(entity4, entity5);
     batchWriter.put(ENTITY3, entity1, entity2);
     batchWriter.submit();
@@ -312,9 +313,9 @@ public void testRunGqlQueryNoCasting() throws DatastoreException {
         Type.PROJECTION, "select __key__ from " + KIND1).build();
     QueryResult keyProjectionResult = datastore.run(keyProjectionQuery);
     assertTrue(keyProjectionResult.hasNext());
-    ProjectionEntity p = keyProjectionResult.next();
-    assertEquals(KEY1, p.key());
-    assertTrue(p.properties().isEmpty());
+    ProjectionEntity projectionEntity = keyProjectionResult.next();
+    assertEquals(KEY1, projectionEntity.key());
+    assertTrue(projectionEntity.properties().isEmpty());
     assertFalse(keyProjectionResult.hasNext());
 
     GqlQuery projectionQuery = GqlQuery.builder(
@@ -338,7 +339,7 @@ public void testRunGqlQueryNoCasting() throws DatastoreException {
 
     QueryResult projectionResult = datastore.run(projectionQuery);
     assertTrue(projectionResult.hasNext());
-    ProjectionEntity projectionEntity = projectionResult.next();
+    projectionEntity = projectionResult.next();
     assertEquals("str", projectionEntity.getString("str"));
     assertEquals(DATE_TIME_VALUE.get(), projectionEntity.getDateTime("date"));
     assertEquals(DATE_TIME_VALUE.get().timestampMicroseconds(),
@@ -454,7 +455,7 @@ public void testAllocateId() {
   public void testAllocateIdArray() {
     KeyFactory keyFactory = new KeyFactory(datastore, KIND1);
     PartialKey partialKey1 = keyFactory.newKey();
-    PartialKey partialKey2 = keyFactory.kind(KIND2).addAncestor(KIND1, 10).newKey();
+    PartialKey partialKey2 = keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
     Key key3 = keyFactory.newKey("name");
     Key key4 = keyFactory.newKey(1);
     Iterator result =
@@ -481,14 +482,14 @@ public void testGet() {
     entity = datastore.get(KEY1);
     assertEquals(ENTITY1, entity);
     StringValue value1 = entity.getValue("str");
-    BooleanValue value2 = entity.getValue("bool");
-    ListValue value3 = entity.getValue("list");
-    DateTimeValue value4 = entity.getValue("date");
-    PartialEntity value5 = entity.getEntity("partial1");
     assertEquals(STR_VALUE, value1);
+    BooleanValue value2 = entity.getValue("bool");
     assertEquals(BOOL_VALUE, value2);
+    ListValue value3 = entity.getValue("list");
     assertEquals(LIST_VALUE2, value3);
+    DateTimeValue value4 = entity.getValue("date");
     assertEquals(DATE_TIME_VALUE, value4);
+    PartialEntity value5 = entity.getEntity("partial1");
     assertEquals(PARTIAL_ENTITY1, value5);
     assertEquals(5, entity.names().size());
     assertFalse(entity.contains("bla"));
@@ -511,7 +512,7 @@ public void testGetArray() {
     Entity partial2 = (Entity) entity3.getEntity("partial2");
     assertEquals(partial1, PARTIAL_ENTITY2);
     assertEquals(partial2, ENTITY2);
-    assertEquals(Value.Type.BOOLEAN, entity3.type("bool"));
+    assertEquals(Value.Type.BOOLEAN, entity3.getValue("bool").type());
     assertEquals(6, entity3.names().size());
     assertFalse(entity3.contains("bla"));
     try {
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 706c247c6623..636ab831c8ac 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -23,10 +23,10 @@
 public class SerializationTest {
 
   private static final PartialKey INCOMPLETE_KEY1 =
-      PartialKey.builder("ds", "k").addAncestor("p", 1).build();
+      PartialKey.builder("ds", "k").ancestors(PathElement.of("p", 1)).build();
   private static final Key KEY1 = Key.builder("ds", "k", "n").build();
   private static final PartialKey INCOMPLETE_KEY2 =
-      PartialKey.builder(KEY1, "v").addAncestor("p", 1).build();
+      PartialKey.builder(KEY1, "v").ancestors(PathElement.of("p", 1)).build();
   private static final Key KEY2 = Key.builder(KEY1, "v", 2).build();
   private static final DateTime DATE_TIME1 = DateTime.now();
   private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world"));

From 01ab2c08244fc60af8995586dba6d2c9b160c884 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 19 Dec 2014 16:30:25 -0800
Subject: [PATCH 069/732] Adding DatastoreHelper

---
 .../com/google/gcloud/datastore/BaseKey.java  |   6 +-
 .../gcloud/datastore/DatastoreHelper.java     | 131 ++++++++++++++++++
 .../gcloud/datastore/DatastoreService.java    |   4 +-
 .../google/gcloud/datastore/KeyFactory.java   |   4 +-
 .../google/gcloud/datastore/PathElement.java  |   2 +-
 .../google/gcloud/datastore/package-info.java |   2 +-
 .../datastore/DatastoreServiceTest.java       |  50 +++----
 7 files changed, 160 insertions(+), 39 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreHelper.java

diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index f9d961034941..ea9f600aff7e 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -32,9 +32,13 @@ abstract static class Builder> {
 
     private static final int MAX_PATH = 100;
 
-    public Builder(String dataset, String kind) {
+    public Builder(String dataset) {
       this.dataset = validateDataset(dataset);
       ancestors = new LinkedList<>();
+    }
+
+    public Builder(String dataset, String kind) {
+      this(dataset);
       this.kind = validateKind(kind);
     }
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
new file mode 100644
index 000000000000..7fe4aeda0d96
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -0,0 +1,131 @@
+package com.google.gcloud.datastore;
+
+import com.google.common.collect.Maps;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Adds some functionality to DatastoreService that should
+ * be provided statically to the interface (Java 8).
+ *
+ */
+public class DatastoreHelper implements DatastoreService {
+
+  private final DatastoreService delegate;
+
+  private DatastoreHelper(DatastoreService delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public Entity get(Key key) {
+    return delegate.get(key);
+  }
+
+  @Override
+  public Iterator get(Key key, Key... others) {
+    return delegate.get(key, others);
+  }
+
+  @Override
+  public  QueryResult run(Query query) {
+    return delegate.run(query);
+  }
+
+  @Override
+  public DatastoreServiceOptions options() {
+    return delegate.options();
+  }
+
+  @Override
+  public Transaction newTransaction(TransactionOption... options) {
+    return delegate.newTransaction(options);
+  }
+
+  @Override
+  public BatchWriter newBatchWriter(BatchWriteOption... options) {
+    return delegate.newBatchWriter(options);
+  }
+
+  @Override
+  public Key allocateId(PartialKey key) {
+    return delegate.allocateId(key);
+  }
+
+  @Override
+  public Iterator allocateId(PartialKey key, PartialKey... others) {
+    return delegate.allocateId(key, others);
+  }
+
+  @Override
+  public void add(Entity... entity) {
+    delegate.add(entity);
+  }
+
+  @Override
+  public void update(Entity... entity) {
+    delegate.update(entity);
+  }
+
+  @Override
+  public void put(Entity... entity) {
+    delegate.put(entity);
+  }
+
+  @Override
+  public void delete(Key... key) {
+    delegate.delete(key);
+  }
+
+  /**
+   * Returns a new KeyFactory for this service
+   */
+  public KeyFactory newKeyFactory() {
+    return new KeyFactory(this);
+  }
+
+  /**
+   * Returns a list with a value for each given key (ordered by input).
+   * A {@code null} would be returned for non-existing keys.
+   */
+  public List fetch(Key key, Key... others) {
+    Iterator entities = delegate.get(key, others);
+    Map map = Maps.newHashMapWithExpectedSize(1 + others.length);
+    while (entities.hasNext()) {
+      Entity entity = entities.next();
+      map.put(entity.key(), entity);
+    }
+    List list = new ArrayList<>(1 + others.length);
+    list.add(map.get(key));
+    for (Key other : others) {
+      list.add(map.get(other));
+    }
+    return list;
+  }
+
+  public interface RunInTransaction {
+    void run(DatastoreReaderWriter readerWriter);
+  }
+
+  public void runInTransaction(RunInTransaction runFor, TransactionOption... options) {
+    Transaction transaction = newTransaction(options);
+    try {
+      runFor.run(transaction);
+      transaction.commit();
+    } finally {
+      if (transaction.active()) {
+        transaction.rollback();
+      }
+    }
+  }
+
+  public static DatastoreHelper createFor(DatastoreService datastoreService) {
+    if (datastoreService instanceof DatastoreHelper) {
+      return (DatastoreHelper) datastoreService;
+    }
+    return new DatastoreHelper(datastoreService);
+  }
+}
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index 45076277f701..f0f84191b18e 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -17,13 +17,13 @@ public interface DatastoreService extends DatastoreReaderWriter {
    *
    * @throws DatastoreServiceExcepiton upon failure
    */
-  Transaction newTransaction(TransactionOption... transactionOption);
+  Transaction newTransaction(TransactionOption... options);
 
   /**
    * Returns a new Batch writer for processing multiple write operations
    * in one request.
    */
-  BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption);
+  BatchWriter newBatchWriter(BatchWriteOption... options);
 
   /**
    * Allocate a unique id for the given key.
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index 4984d71100fe..08d854404a3b 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -12,8 +12,8 @@ public final class KeyFactory extends BaseKey.Builder {
 
   private final DatastoreService service;
 
-  public KeyFactory(DatastoreService service, String kind) {
-    super(checkNotNull(service).options().dataset(), kind);
+  public KeyFactory(DatastoreService service) {
+    super(checkNotNull(service).options().dataset());
     this.service = service;
     namespace(service.options().namespace());
   }
diff --git a/src/main/java/com/google/gcloud/datastore/PathElement.java b/src/main/java/com/google/gcloud/datastore/PathElement.java
index 82a4545faf8e..5652359f2818 100644
--- a/src/main/java/com/google/gcloud/datastore/PathElement.java
+++ b/src/main/java/com/google/gcloud/datastore/PathElement.java
@@ -21,7 +21,7 @@ public final class PathElement extends Serializable
   private final transient String name;
 
   private PathElement(String kind, String name, Long id) {
-    this.kind = checkNotNull(kind);
+    this.kind = checkNotNull(kind, "kind must not be null");
     this.name = name;
     this.id = id;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index ea3e92b7386b..88933a598ca9 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -5,7 +5,7 @@
  * 
 {@code
  * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
- * KeyFactory keyFactory = new KeyFactory(datastore, kind);
+ * KeyFactory keyFactory = new KeyFactory(datastore).kind(kind);
  * Key key = keyFactory.newKey(keyName);
  * Entity entity = datastore.get(key);
  * if (entity == null) {
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index a1d021c3739d..f0df27a88230 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -11,7 +11,6 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.api.services.datastore.client.Datastore;
 import com.google.api.services.datastore.client.DatastoreException;
-import com.google.common.collect.Maps;
 import com.google.gcloud.datastore.Query.Type;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
@@ -21,7 +20,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -68,6 +66,7 @@ public class DatastoreServiceTest {
 
   private DatastoreServiceOptions options;
   private DatastoreService datastore;
+  private DatastoreHelper helper;
 
   @Before
   public void setUp() {
@@ -83,6 +82,7 @@ public void setUp() {
         .host("http://localhost:8080")
         .build();
     datastore = DatastoreServiceFactory.getDefault(options);
+    helper = DatastoreHelper.createFor(datastore);
     // Prepare data for testing
     datastore.delete(KEY1, KEY2, KEY3, KEY4, KEY5);
     datastore.add(ENTITY1, ENTITY2);
@@ -105,7 +105,7 @@ public void testNewTransactionCommit() {
     transaction.delete(KEY1);
     transaction.commit();
 
-    List list = fetch(KEY1, KEY2, KEY3);
+    List list = helper.fetch(KEY1, KEY2, KEY3);
     assertNull(list.get(0));
     assertEquals(entity2, list.get(1));
     assertEquals(ENTITY3, list.get(2));
@@ -197,7 +197,7 @@ public void testNewTransactionRollback() {
 
     verifyNotUsable(transaction);
 
-    List list = fetch(KEY1, KEY2, KEY3);
+    List list = helper.fetch(KEY1, KEY2, KEY3);
     assertEquals(ENTITY1, list.get(0));
     assertEquals(ENTITY2, list.get(1));
     assertNull(list.get(2));
@@ -245,7 +245,8 @@ public void testNewBatchWriter() {
     batchWriter.add(entity4, entity5);
     batchWriter.put(ENTITY3, entity1, entity2);
     batchWriter.submit();
-    Iterator entities = fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
+    Iterator entities =
+        helper.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(entity1, entities.next());
     assertEquals(entity2, entities.next());
     assertEquals(ENTITY3, entities.next());
@@ -265,7 +266,7 @@ public void testNewBatchWriter() {
     batchWriter.delete(entity4.key(), entity5.key());
     batchWriter.update(ENTITY1, ENTITY2, ENTITY3);
     batchWriter.submit();
-    entities = fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
+    entities = helper.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(ENTITY1, entities.next());
     assertEquals(ENTITY2, entities.next());
     assertEquals(ENTITY3, entities.next());
@@ -431,7 +432,7 @@ public void testRunStructuredQuery() throws DatastoreException {
 
   @Test
   public void testAllocateId() {
-    KeyFactory keyFactory = new KeyFactory(datastore, KIND1);
+    KeyFactory keyFactory = helper.newKeyFactory().kind(KIND1);
     PartialKey pk1 = keyFactory.newKey();
     Key key1 = keyFactory.allocateId();
     assertEquals(key1.dataset(), pk1.dataset());
@@ -453,7 +454,7 @@ public void testAllocateId() {
 
   @Test
   public void testAllocateIdArray() {
-    KeyFactory keyFactory = new KeyFactory(datastore, KIND1);
+    KeyFactory keyFactory = helper.newKeyFactory().kind(KIND1);
     PartialKey partialKey1 = keyFactory.newKey();
     PartialKey partialKey2 = keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
     Key key3 = keyFactory.newKey("name");
@@ -499,7 +500,7 @@ public void testGet() {
   public void testGetArray() {
     datastore.put(ENTITY3);
     Iterator result =
-        fetch(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3).iterator();
+        helper.fetch(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3).iterator();
     assertEquals(ENTITY1, result.next());
     assertNull(result.next());
     assertEquals(ENTITY2, result.next());
@@ -525,24 +526,9 @@ public void testGetArray() {
     // TODO(ozarov): construct a test to verify more results
   }
 
-  public List fetch(Key key, Key... others) {
-    Iterator entities = datastore.get(key, others);
-    Map map = Maps.newHashMapWithExpectedSize(1 + others.length);
-    while (entities.hasNext()) {
-      Entity entity = entities.next();
-      map.put(entity.key(), entity);
-    }
-    List list = new ArrayList<>(1 + others.length);
-    list.add(map.get(key));
-    for (Key other : others) {
-      list.add(map.get(other));
-    }
-    return list;
-  }
-
   @Test
   public void testAdd() {
-    List keys = fetch(ENTITY1.key(), ENTITY3.key());
+    List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -559,7 +545,7 @@ public void testAdd() {
 
   @Test
   public void testUpdate() {
-    List keys = fetch(ENTITY1.key(), ENTITY3.key());
+    List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -580,7 +566,7 @@ public void testUpdate() {
 
   @Test
   public void testPut() {
-    Iterator keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
@@ -589,7 +575,7 @@ public void testPut() {
     Entity entity2 = Entity.builder(ENTITY2).clear().set("bla", new NullValue()).build();
     assertNotEquals(ENTITY2, entity2);
     datastore.put(ENTITY3, ENTITY1, entity2);
-    keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(entity2, keys.next());
     assertEquals(ENTITY3, keys.next());
@@ -598,13 +584,13 @@ public void testPut() {
 
   @Test
   public void testDelete() {
-    Iterator keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
     assertFalse(keys.hasNext());
     datastore.delete(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
-    keys = fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertNull(keys.next());
     assertNull(keys.next());
     assertNull(keys.next());
@@ -613,10 +599,10 @@ public void testDelete() {
 
   @Test
   public void testKeyFactory() {
-    KeyFactory keyFactory = new KeyFactory(datastore, KIND1);
+    KeyFactory keyFactory = new KeyFactory(datastore).kind(KIND1);
     assertEquals(PARTIAL_KEY1, keyFactory.newKey());
     assertEquals(PartialKey.builder(PARTIAL_KEY1).kind(KIND2).build(),
-        new KeyFactory(datastore, KIND2).newKey());
+        new KeyFactory(datastore).kind(KIND2).newKey());
     assertEquals(KEY1, keyFactory.newKey("name"));
     assertEquals(Key.builder(KEY1).id(2).build(), keyFactory.newKey(2));
   }

From e6250dccff620e87c7a181e02f4622dfb4f666bc Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sat, 20 Dec 2014 23:53:24 -0800
Subject: [PATCH 070/732] Incorporate retry-helper to datastore service.

---
 .../java/com/google/gcloud/RetryHelper.java   |   5 +-
 .../java/com/google/gcloud/RetryParams.java   |  66 ++---
 .../com/google/gcloud/ServiceOptions.java     |  25 +-
 .../gcloud/datastore/BatchWriterImpl.java     |   2 +-
 .../datastore/DatastoreServiceException.java  |  14 +-
 .../datastore/DatastoreServiceImpl.java       | 134 ++++++---
 .../gcloud/datastore/QueryResultImpl.java     |  19 +-
 .../com/google/gcloud/RetryHelperTest.java    | 256 ++++++++++++++++++
 .../datastore/DatastoreServiceTest.java       |   4 +-
 9 files changed, 440 insertions(+), 85 deletions(-)
 create mode 100644 src/test/java/com/google/gcloud/RetryHelperTest.java

diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index ac4489684e3e..ed0fcc1dab85 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -1,6 +1,7 @@
 package com.google.gcloud;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.Math.max;
 import static java.lang.Math.min;
 import static java.lang.Math.pow;
 import static java.lang.Math.random;
@@ -68,7 +69,7 @@ public static final class RetryInterruptedException extends RetryHelperException
     /**
      * Sets the caller thread interrupt flag and throws {@code RetryInteruptedException}.
      */
-    static void propagate() throws RetryInterruptedException {
+    public static void propagate() throws RetryInterruptedException {
       Thread.currentThread().interrupt();
       throw new RetryInterruptedException();
     }
@@ -200,7 +201,7 @@ static long getSleepDuration(RetryParams retryParams, int attemptsSoFar) {
 
   private static long getExponentialValue(long initialDelay, double backoffFactor, long maxDelay,
       int attemptsSoFar) {
-    return (long) min(maxDelay, pow(backoffFactor, min(1, attemptsSoFar) - 1) * initialDelay);
+    return (long) min(maxDelay, pow(backoffFactor, max(1, attemptsSoFar) - 1) * initialDelay);
   }
 
   public static  V runWithRetries(Callable callable) throws RetryHelperException {
diff --git a/src/main/java/com/google/gcloud/RetryParams.java b/src/main/java/com/google/gcloud/RetryParams.java
index cd500cd4149f..4da4c8399c8e 100644
--- a/src/main/java/com/google/gcloud/RetryParams.java
+++ b/src/main/java/com/google/gcloud/RetryParams.java
@@ -47,37 +47,10 @@ public final class RetryParams implements Serializable {
   private final long totalRetryPeriodMillis;
 
   private static final RetryParams DEFAULT_INSTANCE = new RetryParams(new Builder());
+  private static final RetryParams NO_RETRIES =
+      builder().retryMaxAttempts(1).retryMinAttempts(1).build();
 
 
-  /**
-   * Create a new RetryParams with the parameters from a {@link RetryParams.Builder}
-   *
-   * @param builder the parameters to use to construct the RetryParams object
-   */
-  private RetryParams(Builder builder) {
-    retryMinAttempts = builder.retryMinAttempts;
-    retryMaxAttempts = builder.retryMaxAttempts;
-    initialRetryDelayMillis = builder.initialRetryDelayMillis;
-    maxRetryDelayMillis = builder.maxRetryDelayMillis;
-    retryDelayBackoffFactor = builder.retryDelayBackoffFactor;
-    totalRetryPeriodMillis = builder.totalRetryPeriodMillis;
-    checkArgument(retryMinAttempts >= 0, "retryMinAttempts must not be negative");
-    checkArgument(retryMaxAttempts >= retryMinAttempts,
-        "retryMaxAttempts must not be smaller than retryMinAttempts");
-    checkArgument(initialRetryDelayMillis >= 0, "initialRetryDelayMillis must not be negative");
-    checkArgument(maxRetryDelayMillis >= initialRetryDelayMillis,
-        "maxRetryDelayMillis must not be smaller than initialRetryDelayMillis");
-    checkArgument(retryDelayBackoffFactor >= 0, "retryDelayBackoffFactor must not be negative");
-    checkArgument(totalRetryPeriodMillis >= 0, "totalRetryPeriodMillis must not be negative");
-  }
-
-  /**
-   * Returns an instance with the default parameters.
-   */
-  public static RetryParams getDefaultInstance() {
-    return DEFAULT_INSTANCE;
-  }
-
   /**
    * RetryParams builder.
    */
@@ -188,6 +161,39 @@ public RetryParams build() {
     }
   }
 
+  /**
+   * Create a new RetryParams with the parameters from a {@link RetryParams.Builder}
+   *
+   * @param builder the parameters to use to construct the RetryParams object
+   */
+  private RetryParams(Builder builder) {
+    retryMinAttempts = builder.retryMinAttempts;
+    retryMaxAttempts = builder.retryMaxAttempts;
+    initialRetryDelayMillis = builder.initialRetryDelayMillis;
+    maxRetryDelayMillis = builder.maxRetryDelayMillis;
+    retryDelayBackoffFactor = builder.retryDelayBackoffFactor;
+    totalRetryPeriodMillis = builder.totalRetryPeriodMillis;
+    checkArgument(retryMinAttempts >= 0, "retryMinAttempts must not be negative");
+    checkArgument(retryMaxAttempts >= retryMinAttempts,
+        "retryMaxAttempts must not be smaller than retryMinAttempts");
+    checkArgument(initialRetryDelayMillis >= 0, "initialRetryDelayMillis must not be negative");
+    checkArgument(maxRetryDelayMillis >= initialRetryDelayMillis,
+        "maxRetryDelayMillis must not be smaller than initialRetryDelayMillis");
+    checkArgument(retryDelayBackoffFactor >= 0, "retryDelayBackoffFactor must not be negative");
+    checkArgument(totalRetryPeriodMillis >= 0, "totalRetryPeriodMillis must not be negative");
+  }
+
+  /**
+   * Returns an instance with the default parameters.
+   */
+  public static RetryParams getDefaultInstance() {
+    return DEFAULT_INSTANCE;
+  }
+
+  public static RetryParams noRetries() {
+    return NO_RETRIES;
+  }
+
   /**
    * Returns the retryMinAttempts.
    */
@@ -230,8 +236,6 @@ public long getTotalRetryPeriodMillis() {
     return totalRetryPeriodMillis;
   }
 
-
-
   @Override
   public int hashCode() {
     return Objects.hash(retryMinAttempts, retryMaxAttempts, initialRetryDelayMillis,
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 24510adf7688..0153d364e365 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -17,11 +17,13 @@ public abstract class ServiceOptions {
   private final String host;
   private final HttpTransport httpTransport;
   private final AuthConfig authConfig;
+  private final RetryParams retryParams;
 
   protected ServiceOptions(Builder builder) {
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransport = firstNonNull(builder.httpTransport, defaultHttpTransport());
     authConfig = firstNonNull(builder.authConfig, defaultAuthConfig());
+    retryParams = builder.retryParams;
   }
 
   private static HttpTransport defaultHttpTransport() {
@@ -69,6 +71,7 @@ protected abstract static class Builder> {
     private String host;
     private HttpTransport httpTransport;
     private AuthConfig authConfig;
+    private RetryParams retryParams;
 
     protected Builder() {}
 
@@ -76,26 +79,34 @@ protected Builder(ServiceOptions options) {
       host = options.host;
       httpTransport = options.httpTransport;
       authConfig = options.authConfig;
+      retryParams = options.retryParams;
     }
 
     protected abstract ServiceOptions build();
 
     @SuppressWarnings("unchecked")
+    protected B self() {
+      return (B) this;
+    }
+
     public B host(String host) {
       this.host = host;
-      return (B) this;
+      return self();
     }
 
-    @SuppressWarnings("unchecked")
     public B httpTransport(HttpTransport httpTransport) {
       this.httpTransport = httpTransport;
-      return (B) this;
+      return self();
     }
 
-    @SuppressWarnings("unchecked")
     public B authConfig(AuthConfig authConfig) {
       this.authConfig = authConfig;
-      return (B) this;
+      return self();
+    }
+
+    public B retryParams(RetryParams retryParams) {
+      this.retryParams = retryParams;
+      return self();
     }
   }
 
@@ -113,6 +124,10 @@ public AuthConfig authConfig() {
     return authConfig;
   }
 
+  public RetryParams retryParams() {
+    return retryParams;
+  }
+
   protected HttpRequestInitializer httpRequestInitializer() {
     return authConfig().httpRequestInitializer(httpTransport, scopes());
   }
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
index 9f760faee971..3589fe3fec24 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
@@ -119,7 +119,7 @@ public void submit() {
     }
     DatastoreV1.CommitRequest.Builder requestPb = newCommitRequest();
     requestPb.setMutation(mutationPb);
-    datastore.commitMutation(requestPb);
+    datastore.commit(requestPb.build());
     active = false;
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index 7e5ec384c65c..50c745b7679b 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -3,6 +3,8 @@
 import com.google.api.services.datastore.client.DatastoreException;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
+import com.google.gcloud.RetryHelper;
+import com.google.gcloud.RetryHelper.RetryHelperException;
 
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -37,7 +39,7 @@ public enum Code {
 
     Code(boolean isTransient, String msg) {
       this.isTransient = isTransient;
-      this.defaultMessage = msg;
+      defaultMessage = msg;
     }
 
     /**
@@ -77,6 +79,16 @@ public Code code() {
     return code;
   }
 
+  static DatastoreServiceException translateAndThrow(RetryHelperException ex) {
+    if (ex.getCause() instanceof DatastoreException) {
+      return translateAndThrow((DatastoreException) ex.getCause());
+    }
+    if (ex instanceof RetryHelper.RetryInterruptedException) {
+      RetryHelper.RetryInterruptedException.propagate();
+    }
+    throw new DatastoreServiceException(Code.UNKNOWN, ex.getMessage(), ex);
+  }
+
   /**
    * Translate DatastoreException to DatastoreServiceException based on their
    * HTTP error codes. This method will always throw a new DatastoreServiceException.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 91bfa0137b57..875660de74b8 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -3,26 +3,55 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.api.services.datastore.client.Datastore;
 import com.google.api.services.datastore.client.DatastoreException;
+import com.google.common.base.MoreObjects;
 import com.google.common.collect.AbstractIterator;
+import com.google.gcloud.ExceptionHandler;
+import com.google.gcloud.RetryHelper;
+import com.google.gcloud.RetryHelper.RetryHelperException;
+import com.google.gcloud.RetryParams;
 import com.google.protobuf.ByteString;
 
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.concurrent.Callable;
 
 
 final class DatastoreServiceImpl implements DatastoreService {
 
   static final Key[] EMPTY_KEY_ARRAY = {};
   static final PartialKey[] EMPTY_PARTIAL_KEY_ARRAY = {};
+  private static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = 6911242958397733203L;
+
+        @Override
+        public boolean shouldRetry(Exception exception, boolean shouldRetry) {
+          return shouldRetry;
+        }
+
+        @Override
+        public Boolean shouldRetry(Exception exception) {
+          if (exception instanceof DatastoreServiceException) {
+            return ((DatastoreServiceException) exception).code().isTransient();
+          }
+          return null;
+        }
+      };
+  private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
+      .abortOn(RuntimeException.class, DatastoreException.class)
+      .interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
 
   private final DatastoreServiceOptions options;
   private final Datastore datastore;
+  private final RetryParams retryParams;
 
   DatastoreServiceImpl(DatastoreServiceOptions options, Datastore datastore) {
     this.options = options;
     this.datastore = datastore;
+    retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
   }
 
   @Override
@@ -49,10 +78,14 @@  QueryResult run(DatastoreV1.ReadOptions readOptionsPb, Query query) {
     return new QueryResultImpl<>(this, readOptionsPb, query);
   }
 
-  DatastoreV1.RunQueryResponse run(DatastoreV1.RunQueryRequest requestPb) {
+  DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestPb) {
     try {
-      return datastore.runQuery(requestPb);
-    } catch (DatastoreException e) {
+      return RetryHelper.runWithRetries(new Callable() {
+        @Override public DatastoreV1.RunQueryResponse call() throws DatastoreException {
+          return datastore.runQuery(requestPb);
+        }
+      }, retryParams, EXCEPTION_HANDLER);
+    } catch (RetryHelperException e) {
       throw DatastoreServiceException.translateAndThrow(e);
     }
   }
@@ -70,20 +103,26 @@ public Iterator allocateId(PartialKey key, PartialKey... others) {
       requestPb.addKey(trimNameOrId(other).toPb());
     }
     // TODO(ozarov): will need to populate "force" after b/18594027 is fixed.
-    try {
-      DatastoreV1.AllocateIdsResponse responsePb = datastore.allocateIds(requestPb.build());
-      final Iterator keys = responsePb.getKeyList().iterator();
-      return new AbstractIterator() {
+    DatastoreV1.AllocateIdsResponse responsePb = allocateIds(requestPb.build());
+    final Iterator keys = responsePb.getKeyList().iterator();
+    return new AbstractIterator() {
+      @Override protected Key computeNext() {
+        if (keys.hasNext()) {
+          return Key.fromPb(keys.next());
+        }
+        return endOfData();
+      }
+    };
+  }
 
-        @Override
-        protected Key computeNext() {
-          if (keys.hasNext()) {
-            return Key.fromPb(keys.next());
-          }
-          return endOfData();
+  DatastoreV1.AllocateIdsResponse allocateIds(final DatastoreV1.AllocateIdsRequest requestPb) {
+    try {
+      return RetryHelper.runWithRetries(new Callable() {
+        @Override public DatastoreV1.AllocateIdsResponse call() throws DatastoreException {
+          return datastore.allocateIds(requestPb);
         }
-      };
-    } catch (DatastoreException e) {
+      }, retryParams, EXCEPTION_HANDLER);
+    } catch (RetryHelperException e) {
       throw DatastoreServiceException.translateAndThrow(e);
     }
   }
@@ -131,15 +170,11 @@ final class ResultsIterator extends AbstractIterator {
     }
 
     private void loadResults() {
-      try {
-        DatastoreV1.LookupResponse responsePb = datastore.lookup(requestPb.build());
-        iter = responsePb.getFoundList().iterator();
-        requestPb.clearKey();
-        if (responsePb.getDeferredCount() > 0) {
-          requestPb.addAllKey(responsePb.getDeferredList());
-        }
-      } catch (DatastoreException e) {
-        throw DatastoreServiceException.translateAndThrow(e);
+      DatastoreV1.LookupResponse responsePb = lookup(requestPb.build());
+      iter = responsePb.getFoundList().iterator();
+      requestPb.clearKey();
+      if (responsePb.getDeferredCount() > 0) {
+        requestPb.addAllKey(responsePb.getDeferredList());
       }
     }
 
@@ -158,6 +193,18 @@ protected Entity computeNext() {
     }
   }
 
+  DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) {
+    try {
+      return RetryHelper.runWithRetries(new Callable() {
+        @Override public DatastoreV1.LookupResponse call() throws DatastoreException {
+          return datastore.lookup(requestPb);
+        }
+      }, retryParams, EXCEPTION_HANDLER);
+    } catch (RetryHelperException e) {
+      throw DatastoreServiceException.translateAndThrow(e);
+    }
+  }
+
   @Override
   public void add(Entity... entities) {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
@@ -215,23 +262,34 @@ private void commitMutation(DatastoreV1.Mutation.Builder mutationPb) {
     DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
     requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL);
     requestPb.setMutation(mutationPb);
-    commitMutation(requestPb);
+    commit(requestPb.build());
   }
 
-  void commitMutation(DatastoreV1.CommitRequest.Builder requestPb) {
+  DatastoreV1.CommitResponse commit(final DatastoreV1.CommitRequest requestPb) {
     try {
-      datastore.commit(requestPb.build());
-    } catch (DatastoreException e) {
+      return RetryHelper.runWithRetries(new Callable() {
+        @Override public DatastoreV1.CommitResponse call() throws DatastoreException {
+          return datastore.commit(requestPb);
+        }
+      }, retryParams, EXCEPTION_HANDLER);
+    } catch (RetryHelperException e) {
       throw DatastoreServiceException.translateAndThrow(e);
     }
   }
 
   ByteString requestTransactionId(DatastoreV1.BeginTransactionRequest.Builder requestPb) {
+    return beginTransaction(requestPb.build()).getTransaction();
+  }
+
+  DatastoreV1.BeginTransactionResponse beginTransaction(
+      final DatastoreV1.BeginTransactionRequest requestPb) {
     try {
-      DatastoreV1.BeginTransactionResponse responsePb =
-          datastore.beginTransaction(requestPb.build());
-      return responsePb.getTransaction();
-    } catch (DatastoreException e) {
+      return RetryHelper.runWithRetries(new Callable() {
+        @Override public DatastoreV1.BeginTransactionResponse call() throws DatastoreException {
+          return datastore.beginTransaction(requestPb);
+        }
+      }, retryParams, EXCEPTION_HANDLER);
+    } catch (RetryHelperException e) {
       throw DatastoreServiceException.translateAndThrow(e);
     }
   }
@@ -239,9 +297,17 @@ ByteString requestTransactionId(DatastoreV1.BeginTransactionRequest.Builder requ
   void rollbackTransaction(ByteString transaction) {
     DatastoreV1.RollbackRequest.Builder requestPb = DatastoreV1.RollbackRequest.newBuilder();
     requestPb.setTransaction(transaction);
+    rollback(requestPb.build());
+  }
+
+  DatastoreV1.RollbackResponse rollback(final DatastoreV1.RollbackRequest requestPb) {
     try {
-      datastore.rollback(requestPb.build());
-    } catch (DatastoreException e) {
+      return RetryHelper.runWithRetries(new Callable() {
+        @Override public DatastoreV1.RollbackResponse call() throws DatastoreException {
+          return datastore.rollback(requestPb);
+        }
+      }, retryParams, EXCEPTION_HANDLER);
+    } catch (RetryHelperException e) {
       throw DatastoreServiceException.translateAndThrow(e);
     }
   }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index 5639eec89195..5c32f028b6b3 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -1,6 +1,7 @@
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.api.services.datastore.DatastoreV1.QueryResultBatch.MoreResultsType;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.AbstractIterator;
 import com.google.gcloud.datastore.Query.Type;
@@ -15,7 +16,8 @@ class QueryResultImpl extends AbstractIterator implements QueryResult {
   private final Query.Type queryType;
   private Query query;
   private Query.Type actualType;
-  private DatastoreV1.QueryResultBatch resultPb;
+  private DatastoreV1.QueryResultBatch queryResultBatchPb;
+  private boolean lastBatch;
   private Iterator entityResultPbIter;
   //private ByteString cursor; // only available in v1beta3
 
@@ -37,32 +39,31 @@ class QueryResultImpl extends AbstractIterator implements QueryResult {
     sendRequest();
   }
 
-  private DatastoreV1.QueryResultBatch sendRequest() {
+  private void sendRequest() {
     DatastoreV1.RunQueryRequest.Builder requestPb = DatastoreV1.RunQueryRequest.newBuilder();
     if (readOptionsPb != null) {
       requestPb.setReadOptions(readOptionsPb);
     }
     requestPb.setPartitionId(partitionIdPb);
     query.populatePb(requestPb);
-    resultPb = datastore.run(requestPb.build()).getBatch();
-    entityResultPbIter = resultPb.getEntityResultList().iterator();
+    queryResultBatchPb = datastore.runQuery(requestPb.build()).getBatch();
+    lastBatch = queryResultBatchPb.getMoreResults() != MoreResultsType.NOT_FINISHED;
+    entityResultPbIter = queryResultBatchPb.getEntityResultList().iterator();
     // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
-    actualType = Type.fromPb(resultPb.getEntityResultType());
+    actualType = Type.fromPb(queryResultBatchPb.getEntityResultType());
     if (queryType == Type.PROJECTION) {
       // projection entity can represent all type of results
       actualType = Type.PROJECTION;
     }
     Preconditions.checkState(queryType.isAssignableFrom(actualType),
         "Unexpected result type " + actualType + " vs " + queryType);
-    return resultPb;
   }
 
   @SuppressWarnings("unchecked")
   @Override
   protected T computeNext() {
-    while (!entityResultPbIter.hasNext()
-        && resultPb.getMoreResults() == DatastoreV1.QueryResultBatch.MoreResultsType.NOT_FINISHED) {
-      query = query.nextQuery(resultPb);
+    while (!entityResultPbIter.hasNext() && !lastBatch) {
+      query = query.nextQuery(queryResultBatchPb);
       sendRequest();
     }
     if (!entityResultPbIter.hasNext()) {
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/src/test/java/com/google/gcloud/RetryHelperTest.java
new file mode 100644
index 000000000000..565d5680df96
--- /dev/null
+++ b/src/test/java/com/google/gcloud/RetryHelperTest.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2012 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud;
+
+import static java.util.concurrent.Executors.callable;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Ticker;
+import com.google.gcloud.RetryHelper.NonRetriableException;
+import com.google.gcloud.RetryHelper.RetriesExhaustedException;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Tests for {@link RetryHelper}.
+ */
+public class RetryHelperTest {
+
+  @Test
+  public void testTriesWithExceptionHandling() {
+    assertNull(RetryHelper.getContext());
+    RetryParams params =
+        RetryParams.builder().initialRetryDelayMillis(0).retryMaxAttempts(3).build();
+    ExceptionHandler handler = ExceptionHandler.builder()
+        .retryOn(IOException.class).abortOn(RuntimeException.class).build();
+    final AtomicInteger count = new AtomicInteger(3);
+    try {
+      RetryHelper.runWithRetries(new Callable() {
+        @Override public Void call() throws IOException, NullPointerException {
+          if (count.decrementAndGet() == 2) {
+            assertEquals(1, RetryHelper.getContext().getAttemptNumber());
+            throw new IOException("should be retried");
+          }
+          assertEquals(2, RetryHelper.getContext().getAttemptNumber());
+          throw new NullPointerException("Boo!");
+        }
+      }, params, handler);
+      fail("Exception should have been thrown");
+    } catch (NonRetriableException ex) {
+      assertEquals("Boo!", ex.getCause().getMessage());
+      assertEquals(1, count.intValue());
+    }
+    assertNull(RetryHelper.getContext());
+
+    @SuppressWarnings("serial") class E1 extends Exception {}
+    @SuppressWarnings("serial") class E2 extends E1 {}
+    @SuppressWarnings("serial") class E3 extends E1 {}
+    @SuppressWarnings("serial") class E4 extends E2 {}
+
+    params = RetryParams.builder().initialRetryDelayMillis(0).retryMaxAttempts(5).build();
+    handler = ExceptionHandler.builder().retryOn(E1.class, E4.class).abortOn(E3.class).build();
+    final Iterator exceptions =
+        Arrays.asList(new E1(), new E2(), new E4(), new E3()).iterator();
+    try {
+      RetryHelper.runWithRetries(new Callable() {
+        @Override public Void call() throws E1 {
+          E1 exception = exceptions.next();
+          throw exception;
+        }
+      }, params, handler);
+      fail("Exception should have been thrown");
+    } catch (NonRetriableException ex) {
+      assertTrue(ex.getCause() instanceof E3);
+    }
+    assertNull(RetryHelper.getContext());
+  }
+
+  @Test
+  public void testTriesAtLeastMinTimes() {
+    // Total retry period set to 60 seconds so as to not factor into test
+    RetryParams params = RetryParams.builder().initialRetryDelayMillis(0)
+        .totalRetryPeriodMillis(60000)
+        .retryMinAttempts(5)
+        .retryMaxAttempts(10)
+        .build();
+    final int timesToFail = 7;
+    assertNull(RetryHelper.getContext());
+    int attempted = RetryHelper.runWithRetries(new Callable() {
+      int timesCalled = 0;
+      @Override public Integer call() throws IOException {
+        timesCalled++;
+        assertEquals(timesCalled, RetryHelper.getContext().getAttemptNumber());
+        assertEquals(10, RetryHelper.getContext().getRetryParams().getRetryMaxAttempts());
+        if (timesCalled <= timesToFail) {
+          throw new IOException();
+        }
+        return timesCalled;
+      }
+    }, params, ExceptionHandler.getDefaultInstance());
+    assertEquals(timesToFail + 1, attempted);
+    assertNull(RetryHelper.getContext());
+  }
+
+  @Test
+  public void testTriesNoMoreThanMaxTimes() {
+    // Total retry period set to 60 seconds so as to not factor into test
+    final int maxAttempts = 10;
+    RetryParams params = RetryParams.builder().initialRetryDelayMillis(0)
+        .totalRetryPeriodMillis(60000)
+        .retryMinAttempts(0)
+        .retryMaxAttempts(maxAttempts)
+        .build();
+    final AtomicInteger timesCalled = new AtomicInteger(0);
+    try {
+      RetryHelper.runWithRetries(callable(new Runnable() {
+        @Override public void run() {
+          // Throw an exception up to maxAttempts times, should never be called beyond that
+          if (timesCalled.incrementAndGet() <= maxAttempts) {
+            throw new RuntimeException();
+          }
+          fail("Body was executed too many times: " + timesCalled.get());
+        }
+      }), params, ExceptionHandler.builder().retryOn(RuntimeException.class).build());
+      // Unnecessary as this line should not be possible reach even if RetryHandler is broken
+      fail("Should not have succeeded, expected all attempts to fail and give up.");
+    } catch (RetriesExhaustedException expected) {
+      // Expect the body to run exactly maxAttempts times
+      assertEquals(maxAttempts, timesCalled.get());
+    }
+  }
+
+  private class FakeTicker extends Ticker {
+    private final AtomicLong nanos = new AtomicLong();
+
+    // Advances the ticker value by {@code time} in {@code timeUnit}.
+    FakeTicker advance(long time, TimeUnit timeUnit) {
+      return advance(timeUnit.toNanos(time));
+    }
+
+    // Advances the ticker value by {@code nanoseconds}.
+    FakeTicker advance(long nanoseconds) {
+      nanos.addAndGet(nanoseconds);
+      return this;
+    }
+
+    @Override
+    public long read() {
+      return nanos.get();
+    }
+  }
+
+  @Test
+  public void testTriesNoMoreLongerThanTotalRetryPeriod() {
+    final FakeTicker ticker = new FakeTicker();
+    Stopwatch stopwatch = Stopwatch.createUnstarted(ticker);
+    // The 8th attempt (after min and before max) will trigger a 1 second (virtual) delay exceeding
+    // total retry period which is set just under 1 second. Test occurs faster than realtime.
+    RetryParams params = RetryParams.builder().initialRetryDelayMillis(0)
+        .totalRetryPeriodMillis(999)
+        .retryMinAttempts(5)
+        .retryMaxAttempts(10)
+        .build();
+    ExceptionHandler handler = ExceptionHandler.builder().retryOn(RuntimeException.class).build();
+    final int sleepOnAttempt = 8;
+    final AtomicInteger timesCalled = new AtomicInteger(0);
+    try {
+      RetryHelper.runWithRetries(callable(new Runnable() {
+        @Override public void run() {
+          timesCalled.incrementAndGet();
+          if (timesCalled.get() == sleepOnAttempt) {
+            ticker.advance(1000, TimeUnit.MILLISECONDS);
+          }
+          throw new RuntimeException();
+        }
+      }), params, handler, stopwatch);
+      fail();
+    } catch (RetriesExhaustedException e) {
+      assertEquals(sleepOnAttempt, timesCalled.get());
+    }
+  }
+
+  @Test
+  public void testBackoffIsExponential() {
+    // Total retry period set to 60 seconds so as to not factor into test
+    RetryParams params = RetryParams.builder()
+        .initialRetryDelayMillis(10)
+        .maxRetryDelayMillis(10_000_000)
+        .retryDelayBackoffFactor(2)
+        .totalRetryPeriodMillis(60_000)
+        .retryMinAttempts(0)
+        .retryMaxAttempts(100)
+        .build();
+    long sleepDuration = RetryHelper.getSleepDuration(params, 1);
+    assertTrue("" + sleepDuration, sleepDuration < 13 && sleepDuration >= 7);
+    sleepDuration = RetryHelper.getSleepDuration(params, 2);
+    assertTrue("" + sleepDuration, sleepDuration < 25 && sleepDuration >= 15);
+    sleepDuration = RetryHelper.getSleepDuration(params, 3);
+    assertTrue("" + sleepDuration, sleepDuration < 50 && sleepDuration >= 30);
+    sleepDuration = RetryHelper.getSleepDuration(params, 4);
+    assertTrue("" + sleepDuration, sleepDuration < 100 && sleepDuration >= 60);
+    sleepDuration = RetryHelper.getSleepDuration(params, 5);
+    assertTrue("" + sleepDuration, sleepDuration < 200 && sleepDuration >= 120);
+    sleepDuration = RetryHelper.getSleepDuration(params, 6);
+    assertTrue("" + sleepDuration, sleepDuration < 400 && sleepDuration >= 240);
+    sleepDuration = RetryHelper.getSleepDuration(params, 7);
+    assertTrue("" + sleepDuration, sleepDuration < 800 && sleepDuration >= 480);
+    sleepDuration = RetryHelper.getSleepDuration(params, 8);
+    assertTrue("" + sleepDuration, sleepDuration < 1600 && sleepDuration >= 960);
+    sleepDuration = RetryHelper.getSleepDuration(params, 9);
+    assertTrue("" + sleepDuration, sleepDuration < 3200 && sleepDuration >= 1920);
+    sleepDuration = RetryHelper.getSleepDuration(params, 10);
+    assertTrue("" + sleepDuration, sleepDuration < 6400 && sleepDuration >= 3840);
+    sleepDuration = RetryHelper.getSleepDuration(params, 11);
+    assertTrue("" + sleepDuration, sleepDuration < 12800 && sleepDuration >= 7680);
+    sleepDuration = RetryHelper.getSleepDuration(params, 12);
+    assertTrue("" + sleepDuration, sleepDuration < 25600 && sleepDuration >= 15360);
+  }
+
+  @Test
+  public void testNestedUsage() {
+    assertEquals((1 + 3) * 2, invokeNested(3, 2));
+  }
+
+  private int invokeNested(final int level, final int retries) {
+    if (level < 0) {
+      return 0;
+    }
+    return RetryHelper.runWithRetries(new Callable() {
+      @Override
+      public Integer call() throws IOException {
+        if (RetryHelper.getContext().getAttemptNumber() < retries) {
+          throw new IOException();
+        }
+        assertEquals(retries, RetryHelper.getContext().getAttemptNumber());
+        return invokeNested(level - 1, retries) + RetryHelper.getContext().getAttemptNumber();
+      }
+    });
+  }
+
+}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index f0df27a88230..481ee260d500 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -327,7 +327,7 @@ public void testRunGqlQueryNoCasting() throws DatastoreException {
     requestPb.setGqlQuery(projectionQuery.toPb());
     requestPb.setPartitionId(DatastoreV1.PartitionId.newBuilder().setDatasetId(DATASET).build());
     DatastoreV1.RunQueryResponse responsePb =
-        ((DatastoreServiceImpl) datastore).run(requestPb.build());
+        ((DatastoreServiceImpl) datastore).runQuery(requestPb.build());
     DatastoreV1.RunQueryResponse.Builder responsePbBuilder = responsePb.toBuilder();
     responsePbBuilder.getBatchBuilder()
         .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();
@@ -406,7 +406,7 @@ public void testRunStructuredQuery() throws DatastoreException {
     requestPb.setQuery(projectionQuery.toPb());
     requestPb.setPartitionId(DatastoreV1.PartitionId.newBuilder().setDatasetId(DATASET).build());
     DatastoreV1.RunQueryResponse responsePb =
-        ((DatastoreServiceImpl) datastore).run(requestPb.build());
+        ((DatastoreServiceImpl) datastore).runQuery(requestPb.build());
     DatastoreV1.RunQueryResponse.Builder responsePbBuilder = responsePb.toBuilder();
     responsePbBuilder.getBatchBuilder()
         .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();

From 509072d2347d09fc556b91a089006ef4b94f199d Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 28 Dec 2014 16:57:04 -0800
Subject: [PATCH 071/732] work in progress

---
 .classpath                                    |  1 +
 .project                                      | 13 +++++++
 .../com/google/gcloud/ServiceOptions.java     |  8 +++-
 .../java/com/google/gcloud/storage/Acl.java   | 39 +++++++++++++++++++
 .../com/google/gcloud/storage/Bucket.java     |  6 +--
 .../java/com/google/gcloud/storage/Key.java   | 26 +++++++++++++
 .../google/gcloud/storage/StorageObject.java  | 15 +++++++
 .../google/gcloud/storage/StorageService.java |  1 -
 .../gcloud/storage/StorageServiceImpl.java    |  2 +-
 9 files changed, 105 insertions(+), 6 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/Key.java
 create mode 100644 src/main/java/com/google/gcloud/storage/StorageObject.java

diff --git a/.classpath b/.classpath
index d6cf6121af66..dc5384546f80 100644
--- a/.classpath
+++ b/.classpath
@@ -20,6 +20,7 @@
 	
 		
 			
+			
 		
 	
 	
diff --git a/.project b/.project
index fac3e5997ca3..d31c8122faa2 100644
--- a/.project
+++ b/.project
@@ -5,6 +5,11 @@
 	
 	
 	
+		
+			org.eclipse.wst.common.project.facet.core.builder
+			
+			
+		
 		
 			org.eclipse.jdt.core.javabuilder
 			
@@ -30,11 +35,19 @@
 			
 			
 		
+		
+			org.eclipse.wst.validation.validationbuilder
+			
+			
+		
 	
 	
+		org.eclipse.jem.workbench.JavaEMFNature
+		org.eclipse.wst.common.modulecore.ModuleCoreNature
 		org.eclipse.jdt.core.javanature
 		org.eclipse.m2e.core.maven2Nature
 		net.sf.eclipsecs.core.CheckstyleNature
 		edu.umd.cs.findbugs.plugin.eclipse.findbugsNature
+		org.eclipse.wst.common.project.facet.core.nature
 	
 
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 0153d364e365..a86897349bc0 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -63,7 +63,13 @@ private static AuthConfig defaultAuthConfig() {
   }
 
   protected static String appEngineAppId() {
-    return System.getProperty("com.google.appengine.application.id");
+    try {
+      Class apiProxy = Class.forName("com.google.apphosting.api.ApiProxy");
+      Object currentEnv = apiProxy.getMethod("getCurrentEnvironment").invoke(null);
+      return (String) currentEnv.getClass().getMethod("getAppId").invoke(currentEnv);
+    } catch (Exception ex) {
+      return System.getProperty("com.google.appengine.application.id");
+    }
   }
 
   protected abstract static class Builder> {
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 06f2c467bbaf..37cd880958f7 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -2,4 +2,43 @@
 
 public interface Acl {
 
+  public class ProjectTeam {
+    	// ProjectNumber: The project number.
+	//ProjectNumber string `json:"projectNumber,omitempty"`
+
+	// Team: The team. Can be owners, editors, or viewers.
+	//Team string `json:"team,omitempty"`
+  }
+
+  enum Entity {
+    USER_ID("user-userId"),
+    USER_EMAIL("user-emailAddress"),
+    GROUP_ID("group-groupId"),
+    GROUP_EMAIL("group-emailAddress"),
+    ALL_USERS("allUsers"),
+    ALL_AUTHENTICATED_USERS("allAuthenticatedUsers");
+
+    private final String value;
+
+    Entity(String value) {
+      this.value = value;
+    }
+  }
+
+  String domain();
+
+  Entity entity();
+
+  String entityId();
+
+  String email();
+
+  String etag();
+
+  String generation();
+
+
+  ProjectTeam projectTeam();
+
+  String role();
 }
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index f0f9f8f4b52f..2149cafca6f6 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -14,11 +14,11 @@ public interface Bucket {
 
   void updateDefaultObjectAcl();
 
-  Acl acl(String objectName);
 
-  void updateAcl(String objectName, Acl acl);
 
-  void delete(String... objectName);
+
+
+  void delete(Key... objectKey);
 
   void compose(Iterable source, String dest);
 
diff --git a/src/main/java/com/google/gcloud/storage/Key.java b/src/main/java/com/google/gcloud/storage/Key.java
new file mode 100644
index 000000000000..759cc5e9496f
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/Key.java
@@ -0,0 +1,26 @@
+package com.google.gcloud.storage;
+
+public class Key {
+
+  // TODO: add builder, factory method, toURL, from URL, equals,hashCode, toString
+  private final String bucket;
+  private final String name;
+
+  /*
+  Builder() {
+
+  }*/
+
+  Key(String bucket, String name) {
+    this.bucket = bucket;
+    this.name = name;
+  }
+
+  public String bucket() {
+    return bucket;
+  }
+
+  public String name() {
+    return name;
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageObject.java b/src/main/java/com/google/gcloud/storage/StorageObject.java
new file mode 100644
index 000000000000..0fbee66031b8
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/StorageObject.java
@@ -0,0 +1,15 @@
+package com.google.gcloud.storage;
+
+import java.nio.ByteBuffer;
+
+public interface StorageObject {
+
+  // builder will have an option to populate content and set acl, bucket, name,..
+
+  Key key();
+
+  Acl acl();
+
+  ByteBuffer content();
+
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 52e2c879c0f8..47e2af985f3a 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -5,5 +5,4 @@ public interface StorageService {
   Iterable listBuckets();
 
   Bucket getBucket(String bucket);
-
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 135bff666c0f..85c57bea3a9b 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -9,7 +9,7 @@ final class StorageServiceImpl implements StorageService {
 
   StorageServiceImpl(StorageServiceOptions options) {
     this.options = options;
-    this.storage = options.getStorage();
+    storage = options.getStorage();
   }
 
   @Override

From 7a23a3e4c6e0aefe292b375f0c0cf99f41a6d648 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 29 Dec 2014 17:51:41 -0800
Subject: [PATCH 072/732] fix datastore exception handling and auth handling

---
 pom.xml                                       | 23 +++++++----
 .../java/com/google/gcloud/AuthConfig.java    | 29 ++++++++------
 .../com/google/gcloud/ServiceOptions.java     |  2 +-
 .../datastore/DatastoreServiceException.java  | 39 +++++++++++++------
 .../google/gcloud/datastore/PartialKey.java   |  2 +-
 .../google/gcloud/datastore/PathElement.java  |  2 +-
 .../datastore/DatastoreServiceTest.java       |  3 ++
 7 files changed, 68 insertions(+), 32 deletions(-)

diff --git a/pom.xml b/pom.xml
index 1f8cb504a65c..cccce5ebbadf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,8 +21,7 @@
     
       com.google.guava
       guava
-      RELEASE
-      compile
+      18.0
     
     
       com.google.apis
@@ -35,11 +34,17 @@
       google-api-client-appengine
       1.19.0
       compile
+      
+        
+          guava-jdk5
+          com.google.guava
+        
+      
     
     
       junit
       junit
-      RELEASE
+      4.12
       test
     
     
@@ -66,10 +71,10 @@
       v1beta2-rev23-1.19.0
     
     
-    	org.easymock
-    	easymock
-    	3.3
-    	test
+      org.easymock
+      easymock
+      3.3
+      test
     
   
   
@@ -97,6 +102,10 @@
   
   
     
+      
+        maven-jar-plugin
+        2.5
+      
       
         maven-compiler-plugin
         3.1
diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java
index ddae5ffa1c72..67e7b572c133 100644
--- a/src/main/java/com/google/gcloud/AuthConfig.java
+++ b/src/main/java/com/google/gcloud/AuthConfig.java
@@ -1,5 +1,7 @@
 package com.google.gcloud;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.googleapis.compute.ComputeCredential;
 import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
@@ -8,7 +10,6 @@
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.jackson.JacksonFactory;
-import com.google.common.base.Preconditions;
 
 import java.io.IOException;
 import java.security.GeneralSecurityException;
@@ -31,12 +32,14 @@ private static class ServiceAccountAuthConfig extends AuthConfig {
     private final String account;
     private final PrivateKey privateKey;
 
-    public ServiceAccountAuthConfig(String account, PrivateKey privateKey) {
-      this.account = account;
-      this.privateKey = privateKey;
-      if (privateKey != null) {
-        Preconditions.checkArgument(account != null);
-      }
+    ServiceAccountAuthConfig(String account, PrivateKey privateKey) {
+      this.account = checkNotNull(account);
+      this.privateKey = checkNotNull(privateKey);
+    }
+
+    ServiceAccountAuthConfig() {
+      account = null;
+      privateKey = null;
     }
 
     @Override
@@ -44,10 +47,10 @@ protected HttpRequestInitializer httpRequestInitializer(
         HttpTransport transport, Set scopes) {
       GoogleCredential.Builder builder = new GoogleCredential.Builder()
           .setTransport(transport)
-          .setJsonFactory(new JacksonFactory())
-          .setServiceAccountId(account)
-          .setServiceAccountPrivateKey(privateKey);
+          .setJsonFactory(new JacksonFactory());
       if (privateKey != null) {
+        builder.setServiceAccountPrivateKey(privateKey);
+        builder.setServiceAccountId(account);
         builder.setServiceAccountScopes(scopes);
       }
       return builder.build();
@@ -72,10 +75,14 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport ts, Set REASON_TO_CODE;
+  private static final ImmutableMap HTTP_TO_CODE;
 
   private final Code code;
 
@@ -24,22 +28,29 @@ public class DatastoreServiceException extends RuntimeException {
    */
   public enum Code {
 
-    ABORTED(true, "Request aborted"),
-    DEADLINE_EXCEEDED(true, "Deadline exceeded"),
-    UNAVAILABLE(true, "Could not reach service"),
-    FAILED_PRECONDITION(false, "Invalid request"),
-    INVALID_ARGUMENT(false, "Request parameter has an invalid value"),
-    PERMISSION_DENIED(false, "Unauthorized request"),
-    RESOURCE_EXHAUSTED(false, "Quota exceeded"),
-    INTERNAL(false, "Server returned an error"),
-    UNKNOWN(false, "Unknown failure");
+    ABORTED(true, "Request aborted", 409),
+    DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
+    UNAVAILABLE(true, "Could not reach service", 503),
+    FAILED_PRECONDITION(false, "Invalid request", 412),
+    INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
+    PERMISSION_DENIED(false, "Unauthorized request", 403),
+    UNAUTHORIZED(false, "Unauthorized", 401),
+    RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
+    INTERNAL(false, "Server returned an error", 500),
+    UNKNOWN(false, "Unknown failure", -1);
 
     private final boolean isTransient;
     private final String defaultMessage;
+    private final int httpCode;
 
-    Code(boolean isTransient, String msg) {
+    Code(boolean isTransient, String msg, int httpCode) {
       this.isTransient = isTransient;
       defaultMessage = msg;
+      this.httpCode = httpCode;
+    }
+
+    public Integer httpCode() {
+      return httpCode;
     }
 
     /**
@@ -57,10 +68,13 @@ DatastoreServiceException translate(DatastoreException exception, String msg) {
 
   static {
     ImmutableMap.Builder builder = ImmutableMap.builder();
+    Map httpCodes = new HashMap<>();
     for (Code code : Code.values()) {
       builder.put(code.name(), code);
+      httpCodes.put(code.httpCode(), code);
     }
     REASON_TO_CODE = builder.build();
+    HTTP_TO_CODE = ImmutableMap.copyOf(httpCodes);
   }
 
   public DatastoreServiceException(Code code, String msg, Exception cause) {
@@ -108,7 +122,10 @@ static DatastoreServiceException translateAndThrow(DatastoreException exception)
         // ignore - will be converted to unknown
       }
     }
-    Code code = MoreObjects.firstNonNull(REASON_TO_CODE.get(reason), Code.UNKNOWN);
+    Code code = REASON_TO_CODE.get(reason);
+    if (code == null) {
+      code = MoreObjects.firstNonNull(HTTP_TO_CODE.get(exception.getCode()), Code.UNKNOWN);
+    }
     throw code.translate(exception, message);
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java
index 302abbd065c8..e819aab8ec0b 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java
@@ -1,7 +1,7 @@
 package com.google.gcloud.datastore;
 
-import com.google.api.client.repackaged.com.google.common.base.Preconditions;
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.InvalidProtocolBufferException;
 
diff --git a/src/main/java/com/google/gcloud/datastore/PathElement.java b/src/main/java/com/google/gcloud/datastore/PathElement.java
index 5652359f2818..9e84764e55f7 100644
--- a/src/main/java/com/google/gcloud/datastore/PathElement.java
+++ b/src/main/java/com/google/gcloud/datastore/PathElement.java
@@ -1,6 +1,6 @@
 package com.google.gcloud.datastore;
 
-import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 481ee260d500..a693446afc1e 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -19,6 +19,8 @@
 import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
 
 import java.util.Collections;
 import java.util.HashMap;
@@ -26,6 +28,7 @@
 import java.util.List;
 import java.util.Map;
 
+@RunWith(JUnit4.class)
 public class DatastoreServiceTest {
 
   private static final String DATASET = "dataset1";

From ede68673eb8d66a7404454e84ba2e1f0b6d5bc97 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 30 Dec 2014 19:48:42 -0800
Subject: [PATCH 073/732] Remove the need to provide a full application name
 for dataset (s~ | e~).

---
 .checkstyle                                   |   2 +-
 .eclipse-pmd                                  |   7 +
 .project                                      |  12 +
 checkstyle.xml                                |   8 +-
 full-pmd-ruleset.xml                          |  46 +++
 pmd.xml                                       | 336 ++++++++++++++++++
 .../com/google/gcloud/ExceptionHandler.java   |  59 +--
 .../com/google/gcloud/ServiceOptions.java     | 109 +++---
 .../datastore/DatastoreServiceImpl.java       |  11 +-
 .../datastore/DatastoreServiceOptions.java    |  84 +++--
 .../java/com/google/gcloud/storage/Acl.java   |   8 +-
 .../google/gcloud/ExceptionHandlerTest.java   |  12 +-
 .../com/google/gcloud/RetryHelperTest.java    |  17 +-
 .../datastore/DatastoreServiceTest.java       |   8 +
 14 files changed, 609 insertions(+), 110 deletions(-)
 create mode 100644 .eclipse-pmd
 create mode 100644 full-pmd-ruleset.xml
 create mode 100644 pmd.xml

diff --git a/.checkstyle b/.checkstyle
index 5783bc0d77a1..429677b11adc 100644
--- a/.checkstyle
+++ b/.checkstyle
@@ -1,7 +1,7 @@
 
 
 
-  
+  
     
   
 
diff --git a/.eclipse-pmd b/.eclipse-pmd
new file mode 100644
index 000000000000..c14648afb674
--- /dev/null
+++ b/.eclipse-pmd
@@ -0,0 +1,7 @@
+
+
+  
+  
+    
+  
+
\ No newline at end of file
diff --git a/.project b/.project
index d31c8122faa2..0443050b010d 100644
--- a/.project
+++ b/.project
@@ -40,6 +40,16 @@
 			
 			
 		
+		
+			ntut.csie.rleht.builder.RLBuilder
+			
+			
+		
+		
+			ch.acanda.eclipse.pmd.builder.PMDBuilder
+			
+			
+		
 	
 	
 		org.eclipse.jem.workbench.JavaEMFNature
@@ -49,5 +59,7 @@
 		net.sf.eclipsecs.core.CheckstyleNature
 		edu.umd.cs.findbugs.plugin.eclipse.findbugsNature
 		org.eclipse.wst.common.project.facet.core.nature
+		ntut.csie.rleht.builder.RLNature
+		ch.acanda.eclipse.pmd.builder.PMDNature
 	
 
diff --git a/checkstyle.xml b/checkstyle.xml
index a3c9eecc8e46..814915090b25 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -118,10 +118,12 @@ Checkstyle configurartion that checks the Google coding conventions (https://goo
     
     
     
+      
       
       
       
       
+      
     
     
     
@@ -157,7 +159,11 @@ Checkstyle configurartion that checks the Google coding conventions (https://goo
       
     
     
-    
+    
+      
+      
+      
+    
   
   
     
diff --git a/full-pmd-ruleset.xml b/full-pmd-ruleset.xml
new file mode 100644
index 000000000000..9f1ec73cc736
--- /dev/null
+++ b/full-pmd-ruleset.xml
@@ -0,0 +1,46 @@
+
+
+   Full 5.1.1 PMD rule set
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+
diff --git a/pmd.xml b/pmd.xml
new file mode 100644
index 000000000000..0b206c082bb3
--- /dev/null
+++ b/pmd.xml
@@ -0,0 +1,336 @@
+
+
+   PMD Plugin preferences rule set
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+   
+
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index 9ad46f9d1040..3cb6941d428c 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -1,5 +1,6 @@
 package com.google.gcloud;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -7,6 +8,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
+import com.google.gcloud.ExceptionHandler.Interceptor.RetryResult;
 
 import java.io.Serializable;
 import java.lang.reflect.Method;
@@ -30,25 +32,42 @@ public final class ExceptionHandler implements Serializable {
 
   public interface Interceptor extends Serializable {
 
+    enum RetryResult {
+
+      RETRY(true),
+      ABORT(false);
+
+      private final boolean booleanValue;
+
+      private RetryResult(boolean booleanValue) {
+        this.booleanValue = booleanValue;
+      }
+
+      boolean booleanValue() {
+        return booleanValue;
+      }
+    }
+
     /**
-     * This method is called before evaluating if the exception should be propagated
-     * and could short-circuit the evaluation process.
+     * This method is called before exception evaluation and could short-circuit the process.
      *
      * @param exception the exception that is being evaluated
-     * @return {@code Boolean.TRUE} if exception should be ignored, {@code Boolean.FALSE}
-     *      if exception should be propagated or {@code null} if evaluation should proceed.
+     * @return {@link RetryResult} to indicate if the exception should be ignored
+     *     ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}),
+     *     or evaluation should proceed ({@code null}).
      */
-    Boolean shouldRetry(Exception exception);
+    RetryResult shouldRetry(Exception exception);
 
     /**
-     * This method is called after the evaluation but could alter the result if desired.
+     * This method is called after the evaluation and could alter its result.
      *
      * @param exception the exception that is being evaluated
-     * @param shouldRetry the result of the evaluation
-     * @return {@code true} if exception should be ignored or {@code false}
-     *      if exception should be propagated.
+     * @param retryResult the result of the evaluation so far.
+     * @return {@link RetryResult} to indicate if the exception should be ignored
+     *     ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}),
+     *     or evaluation should proceed ({@code null}).
      */
-    boolean shouldRetry(Exception exception, boolean shouldRetry);
+    RetryResult shouldRetry(Exception exception, RetryResult retryResult);
   }
 
   /**
@@ -121,10 +140,10 @@ static final class RetryInfo implements Serializable {
 
     private static final long serialVersionUID = -4264634837841455974L;
     private final Class exception;
-    private final boolean retry;
+    private final RetryResult retry;
     private final Set children = Sets.newHashSet();
 
-    RetryInfo(Class exception, boolean retry) {
+    RetryInfo(Class exception, RetryResult retry) {
       this.exception = checkNotNull(exception);
       this.retry = retry;
     }
@@ -155,10 +174,10 @@ private ExceptionHandler(Builder builder) {
         Sets.intersection(retriableExceptions, nonRetriableExceptions).isEmpty(),
         "Same exception was found in both retriable and non-retriable sets");
     for (Class exception : retriableExceptions) {
-      addToRetryInfos(retryInfos, new RetryInfo(exception, true));
+      addToRetryInfos(retryInfos, new RetryInfo(exception, RetryResult.RETRY));
     }
     for (Class exception : nonRetriableExceptions) {
-      addToRetryInfos(retryInfos,  new RetryInfo(exception, false));
+      addToRetryInfos(retryInfos,  new RetryInfo(exception, RetryResult.ABORT));
     }
   }
 
@@ -223,17 +242,17 @@ public Set> getNonRetriableExceptions() {
 
   boolean shouldRetry(Exception ex) {
     for (Interceptor interceptor : interceptors) {
-      Boolean shouldRetry = interceptor.shouldRetry(ex);
-      if (shouldRetry != null) {
-        return shouldRetry.booleanValue();
+      RetryResult retryResult = interceptor.shouldRetry(ex);
+      if (retryResult != null) {
+        return retryResult.booleanValue();
       }
     }
     RetryInfo retryInfo = findMostSpecificRetryInfo(retryInfos, ex.getClass());
-    boolean shouldRetry = retryInfo == null ? false : retryInfo.retry;
+    RetryResult retryResult = retryInfo == null ? RetryResult.ABORT : retryInfo.retry;
     for (Interceptor interceptor : interceptors) {
-      shouldRetry = interceptor.shouldRetry(ex, shouldRetry);
+      retryResult = firstNonNull(interceptor.shouldRetry(ex, retryResult), retryResult);
     }
-    return shouldRetry;
+    return retryResult.booleanValue();
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 93c9af4b6136..c23d754a5c46 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -8,6 +8,11 @@
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
 import java.util.Set;
 
 public abstract class ServiceOptions {
@@ -19,6 +24,50 @@ public abstract class ServiceOptions {
   private final AuthConfig authConfig;
   private final RetryParams retryParams;
 
+  protected abstract static class Builder> {
+
+    private String host;
+    private HttpTransport httpTransport;
+    private AuthConfig authConfig;
+    private RetryParams retryParams;
+
+    protected Builder() {}
+
+    protected Builder(ServiceOptions options) {
+      host = options.host;
+      httpTransport = options.httpTransport;
+      authConfig = options.authConfig;
+      retryParams = options.retryParams;
+    }
+
+    protected abstract ServiceOptions build();
+
+    @SuppressWarnings("unchecked")
+    protected B self() {
+      return (B) this;
+    }
+
+    public B host(String host) {
+      this.host = host;
+      return self();
+    }
+
+    public B httpTransport(HttpTransport httpTransport) {
+      this.httpTransport = httpTransport;
+      return self();
+    }
+
+    public B authConfig(AuthConfig authConfig) {
+      this.authConfig = authConfig;
+      return self();
+    }
+
+    public B retryParams(RetryParams retryParams) {
+      this.retryParams = retryParams;
+      return self();
+    }
+  }
+
   protected ServiceOptions(Builder builder) {
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransport = firstNonNull(builder.httpTransport, defaultHttpTransport());
@@ -63,56 +112,20 @@ private static AuthConfig defaultAuthConfig() {
   }
 
   protected static String appEngineAppId() {
-    try {
-      Class apiProxy = Class.forName("com.google.apphosting.api.ApiProxy");
-      Object currentEnv = apiProxy.getMethod("getCurrentEnvironment").invoke(null);
-      return (String) currentEnv.getClass().getMethod("getAppId").invoke(currentEnv);
-    } catch (Exception ex) {
-      return System.getProperty("com.google.appengine.application.id");
-    }
+    return System.getProperty("com.google.appengine.application.id");
   }
 
-  protected abstract static class Builder> {
-
-    private String host;
-    private HttpTransport httpTransport;
-    private AuthConfig authConfig;
-    private RetryParams retryParams;
-
-    protected Builder() {}
-
-    protected Builder(ServiceOptions options) {
-      host = options.host;
-      httpTransport = options.httpTransport;
-      authConfig = options.authConfig;
-      retryParams = options.retryParams;
-    }
-
-    protected abstract ServiceOptions build();
-
-    @SuppressWarnings("unchecked")
-    protected B self() {
-      return (B) this;
-    }
-
-    public B host(String host) {
-      this.host = host;
-      return self();
-    }
-
-    public B httpTransport(HttpTransport httpTransport) {
-      this.httpTransport = httpTransport;
-      return self();
-    }
-
-    public B authConfig(AuthConfig authConfig) {
-      this.authConfig = authConfig;
-      return self();
-    }
-
-    public B retryParams(RetryParams retryParams) {
-      this.retryParams = retryParams;
-      return self();
+  protected static String googleCloudProjectId() {
+    try {
+      URL url = new URL("http://metadata/computeMetadata/v1/project/project-id");
+      URLConnection connection = url.openConnection();
+      connection.setRequestProperty("X-Google-Metadata-Request", "True");
+      try (BufferedReader reader =
+          new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
+        return reader.readLine();
+      }
+    } catch (IOException e) {
+      return null;
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 875660de74b8..a887c03076df 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -28,14 +28,17 @@ final class DatastoreServiceImpl implements DatastoreService {
         private static final long serialVersionUID = 6911242958397733203L;
 
         @Override
-        public boolean shouldRetry(Exception exception, boolean shouldRetry) {
-          return shouldRetry;
+        public RetryResult shouldRetry(Exception exception, RetryResult retryResult) {
+          return null;
         }
 
         @Override
-        public Boolean shouldRetry(Exception exception) {
+        public RetryResult shouldRetry(Exception exception) {
           if (exception instanceof DatastoreServiceException) {
-            return ((DatastoreServiceException) exception).code().isTransient();
+            boolean isTransient = ((DatastoreServiceException) exception).code().isTransient();
+            return isTransient
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.ABORT;
           }
           return null;
         }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 3b81cac646c4..f3cc48d69801 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -1,21 +1,28 @@
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.gcloud.datastore.Validator.validateDataset;
 import static com.google.gcloud.datastore.Validator.validateNamespace;
 
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.services.datastore.DatastoreV1;
+import com.google.api.services.datastore.DatastoreV1.EntityResult;
+import com.google.api.services.datastore.DatastoreV1.LookupResponse;
 import com.google.api.services.datastore.client.Datastore;
+import com.google.api.services.datastore.client.DatastoreException;
 import com.google.api.services.datastore.client.DatastoreFactory;
 import com.google.api.services.datastore.client.DatastoreOptions;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.gcloud.ServiceOptions;
 
 import java.lang.reflect.Method;
+import java.util.Iterator;
 import java.util.Set;
 
 public class DatastoreServiceOptions extends ServiceOptions {
 
+  private static final String DATASET_ENV_NAME = "DATASTORE_DATASET";
   private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore";
   private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email";
   private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE);
@@ -24,15 +31,6 @@ public class DatastoreServiceOptions extends ServiceOptions {
   private final boolean force;
   private final Datastore datastore;
 
-  DatastoreServiceOptions(Builder builder) {
-    super(builder);
-    dataset = firstNonNull(builder.dataset, appEngineAppId());
-    checkArgument(dataset != null, "missing dataset");
-    namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
-    force = builder.force;
-    datastore = builder.datastore;
-  }
-
   public static class Builder extends ServiceOptions.Builder {
 
     private String dataset;
@@ -54,7 +52,7 @@ public DatastoreServiceOptions build() {
       return new DatastoreServiceOptions(this);
     }
 
-    Builder datastore(Datastore datastore) {
+    public Builder datastore(Datastore datastore) {
       this.datastore = datastore;
       return this;
     }
@@ -75,6 +73,58 @@ public Builder force(boolean force) {
     }
   }
 
+  DatastoreServiceOptions(Builder builder) {
+    super(builder);
+    namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
+    force = builder.force;
+
+    // Replace provided dataset with full dataset (s~xxx, e~xxx,...)
+    String tempDataset = firstNonNull(builder.dataset, defaultDataset());
+    Datastore tempDatastore = firstNonNull(builder.datastore,
+        defaultDatastore(tempDataset, host(), httpRequestInitializer()));
+    DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
+    DatastoreV1.Key key = DatastoreV1.Key.newBuilder()
+        .addPathElement(DatastoreV1.Key.PathElement.newBuilder().setKind("__foo__").setName("bar"))
+        .build();
+    requestPb.addKey(key);
+    try {
+      LookupResponse responsePb = tempDatastore.lookup(requestPb.build());
+      if (responsePb.getDeferredCount() > 0) {
+        key = responsePb.getDeferred(0);
+      } else {
+        Iterator combinedIter =
+            Iterables.concat(responsePb.getMissingList(), responsePb.getFoundList()).iterator();
+        key = combinedIter.next().getEntity().getKey();
+      }
+      dataset = key.getPartitionId().getDatasetId();
+      if (builder.datastore == null && !dataset.equals(tempDataset)) {
+        datastore = defaultDatastore(dataset, host(), httpRequestInitializer());
+      } else {
+        datastore = tempDatastore;
+      }
+    } catch (DatastoreException e) {
+      throw DatastoreServiceException.translateAndThrow(e);
+    }
+  }
+
+  private static Datastore defaultDatastore(
+      String dataset, String host, HttpRequestInitializer initializer) {
+    DatastoreOptions options = new DatastoreOptions.Builder()
+        .dataset(dataset)
+        .host(host)
+        .initializer(initializer)
+        .build();
+    return DatastoreFactory.get().create(options);
+  }
+
+  private static String defaultDataset() {
+    String dataset = System.getProperty(DATASET_ENV_NAME, System.getenv(DATASET_ENV_NAME));
+    if (dataset == null) {
+      dataset = appEngineAppId();
+    }
+    return dataset != null ? dataset : googleCloudProjectId();
+  }
+
   public String dataset() {
     return dataset;
   }
@@ -110,16 +160,8 @@ public Builder toBuilder() {
     return new Builder(this);
   }
 
-  private DatastoreOptions toDatastoreOptions() {
-    return new DatastoreOptions.Builder()
-        .dataset(dataset())
-        .host(host())
-        .initializer(httpRequestInitializer())
-        .build();
-  }
-
-  Datastore datastore() {
-    return datastore != null ? datastore : DatastoreFactory.get().create(toDatastoreOptions());
+  public Datastore datastore() {
+    return datastore;
   }
 
   public static Builder builder() {
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 37cd880958f7..13bbe2bfb44d 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -3,11 +3,11 @@
 public interface Acl {
 
   public class ProjectTeam {
-    	// ProjectNumber: The project number.
-	//ProjectNumber string `json:"projectNumber,omitempty"`
+      // ProjectNumber: The project number.
+  //ProjectNumber string `json:"projectNumber,omitempty"`
 
-	// Team: The team. Can be owners, editors, or viewers.
-	//Team string `json:"team,omitempty"`
+  // Team: The team. Can be owners, editors, or viewers.
+  //Team string `json:"team,omitempty"`
   }
 
   enum Entity {
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
index 42905c85a08a..35f9613c42ae 100644
--- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -5,6 +5,7 @@
 import static org.junit.Assert.fail;
 
 import com.google.gcloud.ExceptionHandler.Interceptor;
+import com.google.gcloud.ExceptionHandler.Interceptor.RetryResult;
 
 import org.junit.Test;
 
@@ -112,16 +113,15 @@ public void testShouldTry() {
     assertFalse(handler.shouldRetry(new RuntimeException()));
     assertTrue(handler.shouldRetry(new NullPointerException()));
 
-    final AtomicReference before = new AtomicReference<>(false);
-
+    final AtomicReference before = new AtomicReference<>(RetryResult.ABORT);
     Interceptor interceptor = new Interceptor() {
       @Override
-      public boolean shouldRetry(Exception exception, boolean shouldRetry) {
-        return !shouldRetry;
+      public RetryResult shouldRetry(Exception exception, RetryResult retryResult) {
+        return retryResult == RetryResult.ABORT ? RetryResult.RETRY : RetryResult.ABORT;
       }
 
       @Override
-      public Boolean shouldRetry(Exception exception) {
+      public RetryResult shouldRetry(Exception exception) {
         return before.get();
       }
     };
@@ -134,7 +134,7 @@ public Boolean shouldRetry(Exception exception) {
     assertFalse(handler.shouldRetry(new RuntimeException()));
     assertFalse(handler.shouldRetry(new NullPointerException()));
 
-    before.set(true);
+    before.set(RetryResult.RETRY);
     assertTrue(handler.shouldRetry(new IOException()));
     assertTrue(handler.shouldRetry(new ClosedByInterruptException()));
     assertTrue(handler.shouldRetry(new InterruptedException()));
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/src/test/java/com/google/gcloud/RetryHelperTest.java
index 565d5680df96..257dc892d229 100644
--- a/src/test/java/com/google/gcloud/RetryHelperTest.java
+++ b/src/test/java/com/google/gcloud/RetryHelperTest.java
@@ -68,10 +68,17 @@ public void testTriesWithExceptionHandling() {
     }
     assertNull(RetryHelper.getContext());
 
-    @SuppressWarnings("serial") class E1 extends Exception {}
-    @SuppressWarnings("serial") class E2 extends E1 {}
-    @SuppressWarnings("serial") class E3 extends E1 {}
-    @SuppressWarnings("serial") class E4 extends E2 {}
+    @SuppressWarnings("serial")
+    class E1 extends Exception {}
+
+    @SuppressWarnings("serial")
+    class E2 extends E1 {}
+
+    @SuppressWarnings("serial")
+    class E3 extends E1 {}
+
+    @SuppressWarnings("serial")
+    class E4 extends E2 {}
 
     params = RetryParams.builder().initialRetryDelayMillis(0).retryMaxAttempts(5).build();
     handler = ExceptionHandler.builder().retryOn(E1.class, E4.class).abortOn(E3.class).build();
@@ -253,4 +260,4 @@ public Integer call() throws IOException {
     });
   }
 
-}
\ No newline at end of file
+}
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index a693446afc1e..dd655f884159 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -335,6 +335,10 @@ public void testRunGqlQueryNoCasting() throws DatastoreException {
     responsePbBuilder.getBatchBuilder()
         .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();
     Datastore mockDatastore = EasyMock.createMock(Datastore.class);
+    DatastoreV1.EntityResult found =
+        DatastoreV1.EntityResult.newBuilder().setEntity(ENTITY1.toPb()).build();
+    EasyMock.expect(mockDatastore.lookup(EasyMock.anyObject()))
+        .andReturn(DatastoreV1.LookupResponse.newBuilder().addFound(found).build());
     EasyMock.expect(mockDatastore.runQuery(requestPb.build())).andReturn(responsePbBuilder.build());
     EasyMock.replay(mockDatastore);
     datastore = DatastoreServiceFactory.getDefault(
@@ -414,6 +418,10 @@ public void testRunStructuredQuery() throws DatastoreException {
     responsePbBuilder.getBatchBuilder()
         .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();
     Datastore mockDatastore = EasyMock.createMock(Datastore.class);
+    DatastoreV1.EntityResult missing =
+        DatastoreV1.EntityResult.newBuilder().setEntity(ENTITY1.toPb()).build();
+    EasyMock.expect(mockDatastore.lookup(EasyMock.anyObject()))
+        .andReturn(DatastoreV1.LookupResponse.newBuilder().addMissing(missing).build());
     EasyMock.expect(mockDatastore.runQuery(requestPb.build())).andReturn(responsePbBuilder.build());
     EasyMock.replay(mockDatastore);
     datastore = DatastoreServiceFactory.getDefault(

From 938b04db4a2e034b976ee52abd241f51fa001ca4 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 30 Dec 2014 23:38:56 -0800
Subject: [PATCH 074/732] Make maven happy

---
 .checkstyle                                        |  2 +-
 .project                                           |  8 ++++----
 .settings/org.eclipse.jdt.core.prefs               |  3 +++
 pom.xml                                            |  2 +-
 .../java/com/google/gcloud/ExceptionHandler.java   | 14 +++++++-------
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/.checkstyle b/.checkstyle
index 429677b11adc..5783bc0d77a1 100644
--- a/.checkstyle
+++ b/.checkstyle
@@ -1,7 +1,7 @@
 
 
 
-  
+  
     
   
 
diff --git a/.project b/.project
index 0443050b010d..89b7470af643 100644
--- a/.project
+++ b/.project
@@ -31,22 +31,22 @@
 			
 		
 		
-			org.eclipse.m2e.core.maven2Builder
+			org.eclipse.wst.validation.validationbuilder
 			
 			
 		
 		
-			org.eclipse.wst.validation.validationbuilder
+			ntut.csie.rleht.builder.RLBuilder
 			
 			
 		
 		
-			ntut.csie.rleht.builder.RLBuilder
+			ch.acanda.eclipse.pmd.builder.PMDBuilder
 			
 			
 		
 		
-			ch.acanda.eclipse.pmd.builder.PMDBuilder
+			org.eclipse.m2e.core.maven2Builder
 			
 			
 		
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
index 62f72418e536..83d343fa7442 100644
--- a/.settings/org.eclipse.jdt.core.prefs
+++ b/.settings/org.eclipse.jdt.core.prefs
@@ -8,9 +8,11 @@ org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonN
 org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault
 org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable
 org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
 org.eclipse.jdt.core.compiler.compliance=1.7
 org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
 org.eclipse.jdt.core.compiler.problem.autoboxing=ignore
 org.eclipse.jdt.core.compiler.problem.comparingIdentical=error
 org.eclipse.jdt.core.compiler.problem.deadCode=warning
@@ -19,6 +21,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
 org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
 org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
 org.eclipse.jdt.core.compiler.problem.emptyStatement=warning
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
 org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=warning
 org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
 org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled
diff --git a/pom.xml b/pom.xml
index cccce5ebbadf..4a01c702f696 100644
--- a/pom.xml
+++ b/pom.xml
@@ -108,7 +108,7 @@
       
       
         maven-compiler-plugin
-        3.1
+        3.2
         
           1.7
           1.7
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index 3cb6941d428c..c3b558e58100 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -8,7 +8,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
-import com.google.gcloud.ExceptionHandler.Interceptor.RetryResult;
 
 import java.io.Serializable;
 import java.lang.reflect.Method;
@@ -140,10 +139,10 @@ static final class RetryInfo implements Serializable {
 
     private static final long serialVersionUID = -4264634837841455974L;
     private final Class exception;
-    private final RetryResult retry;
+    private final Interceptor.RetryResult retry;
     private final Set children = Sets.newHashSet();
 
-    RetryInfo(Class exception, RetryResult retry) {
+    RetryInfo(Class exception, Interceptor.RetryResult retry) {
       this.exception = checkNotNull(exception);
       this.retry = retry;
     }
@@ -174,10 +173,10 @@ private ExceptionHandler(Builder builder) {
         Sets.intersection(retriableExceptions, nonRetriableExceptions).isEmpty(),
         "Same exception was found in both retriable and non-retriable sets");
     for (Class exception : retriableExceptions) {
-      addToRetryInfos(retryInfos, new RetryInfo(exception, RetryResult.RETRY));
+      addToRetryInfos(retryInfos, new RetryInfo(exception, Interceptor.RetryResult.RETRY));
     }
     for (Class exception : nonRetriableExceptions) {
-      addToRetryInfos(retryInfos,  new RetryInfo(exception, RetryResult.ABORT));
+      addToRetryInfos(retryInfos,  new RetryInfo(exception, Interceptor.RetryResult.ABORT));
     }
   }
 
@@ -242,13 +241,14 @@ public Set> getNonRetriableExceptions() {
 
   boolean shouldRetry(Exception ex) {
     for (Interceptor interceptor : interceptors) {
-      RetryResult retryResult = interceptor.shouldRetry(ex);
+      Interceptor.RetryResult retryResult = interceptor.shouldRetry(ex);
       if (retryResult != null) {
         return retryResult.booleanValue();
       }
     }
     RetryInfo retryInfo = findMostSpecificRetryInfo(retryInfos, ex.getClass());
-    RetryResult retryResult = retryInfo == null ? RetryResult.ABORT : retryInfo.retry;
+    Interceptor.RetryResult retryResult =
+        retryInfo == null ? Interceptor.RetryResult.ABORT : retryInfo.retry;
     for (Interceptor interceptor : interceptors) {
       retryResult = firstNonNull(interceptor.shouldRetry(ex, retryResult), retryResult);
     }

From 823eb9f76aea833253432af4412f403c62e2b9b5 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 2 Jan 2015 17:50:22 -0800
Subject: [PATCH 075/732] Automatically invoke gcd for testing

---
 .eclipse-pmd                                  |   2 +-
 full-pmd-ruleset.xml                          |  46 ----
 pmd.xml                                       |   4 +-
 pom.xml                                       |  73 ++++++
 ...a => DatastoreServiceIntegrationTest.java} |  28 ++-
 .../gcloud/datastore/LocalGcdHelper.java      | 232 ++++++++++++++++++
 6 files changed, 324 insertions(+), 61 deletions(-)
 delete mode 100644 full-pmd-ruleset.xml
 rename src/test/java/com/google/gcloud/datastore/{DatastoreServiceTest.java => DatastoreServiceIntegrationTest.java} (97%)
 create mode 100644 src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java

diff --git a/.eclipse-pmd b/.eclipse-pmd
index c14648afb674..5b4a4cb75170 100644
--- a/.eclipse-pmd
+++ b/.eclipse-pmd
@@ -2,6 +2,6 @@
 
   
   
-    
+    
   
 
\ No newline at end of file
diff --git a/full-pmd-ruleset.xml b/full-pmd-ruleset.xml
deleted file mode 100644
index 9f1ec73cc736..000000000000
--- a/full-pmd-ruleset.xml
+++ /dev/null
@@ -1,46 +0,0 @@
-
-
-   Full 5.1.1 PMD rule set
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-   
-
diff --git a/pmd.xml b/pmd.xml
index 0b206c082bb3..453c40c8784a 100644
--- a/pmd.xml
+++ b/pmd.xml
@@ -80,7 +80,6 @@
    
    
    
-   
    
    
    
@@ -183,7 +182,6 @@
    
    
    
-   
    
    
    
@@ -333,4 +331,4 @@
    
    
    
-
\ No newline at end of file
+
diff --git a/pom.xml b/pom.xml
index cccce5ebbadf..1dc2f3741739 100644
--- a/pom.xml
+++ b/pom.xml
@@ -102,6 +102,79 @@
   
   
     
+      
+        maven-antrun-plugin
+        1.8
+        
+          
+            before-integration-test
+            pre-integration-test
+            
+              
+                
+                
+                  
+                  
+                    
+                  
+                
+              
+            
+            
+              run
+            
+          
+          
+            after-integration-test
+            post-integration-test
+            
+              
+                
+                
+                  
+                  
+                    
+                  
+                
+              
+            
+            
+              run
+            
+          
+        
+      
+      
+        org.apache.maven.plugins
+        maven-surefire-plugin
+        2.18.1
+        
+          
+            **/*IntegrationTest.java
+          
+        
+      
+      
+        org.apache.maven.plugins
+        maven-failsafe-plugin
+        2.18.1
+        
+          
+            none
+          
+          
+            **/*IntegrationTest.java
+          
+        
+        
+          
+            
+              integration-test
+              verify
+            
+          
+        
+      
       
         maven-jar-plugin
         2.5
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
similarity index 97%
rename from src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
rename to src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index dd655f884159..de43c169c827 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -17,11 +17,13 @@
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
 
 import org.easymock.EasyMock;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -29,9 +31,9 @@
 import java.util.Map;
 
 @RunWith(JUnit4.class)
-public class DatastoreServiceTest {
+public class DatastoreServiceIntegrationTest {
 
-  private static final String DATASET = "dataset1";
+  private static final String DATASET = LocalGcdHelper.DEFAULT_DATASET;
   private static final String KIND1 = "kind1";
   private static final String KIND2 = "kind2";
   private static final NullValue NULL_VALUE = NullValue.of();
@@ -70,19 +72,16 @@ public class DatastoreServiceTest {
   private DatastoreServiceOptions options;
   private DatastoreService datastore;
   private DatastoreHelper helper;
+  private LocalGcdHelper gcdHelper;
 
   @Before
-  public void setUp() {
-    // TODO(ozarov): document that this test depends on a local gcd running.
-    // Unfortunately, the gcd tool is not bundled with the cloud SDK and need
-    // to be downloaded independently from
-    // https://cloud.google.com/datastore/docs/tools/devserver (b/16372095).
-    // To start the gcd run:
-    // gcd.sh create dataset1; gcd.sh start dataset1
-    // We should have an option to start the gcd from maven/ant.
+  public void setUp() throws IOException, InterruptedException {
+    if (!LocalGcdHelper.isActive(DATASET)) {
+      gcdHelper = LocalGcdHelper.start(DATASET);
+    }
     options = DatastoreServiceOptions.builder()
         .dataset(DATASET)
-        .host("http://localhost:8080")
+        .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
     datastore = DatastoreServiceFactory.getDefault(options);
     helper = DatastoreHelper.createFor(datastore);
@@ -91,6 +90,13 @@ public void setUp() {
     datastore.add(ENTITY1, ENTITY2);
   }
 
+  @After
+  public void tearDown() throws IOException, InterruptedException {
+    if (gcdHelper != null) {
+      gcdHelper.stop();
+    }
+  }
+
   @Test
   public void testGetOptions() {
     assertSame(options, datastore.options());
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
new file mode 100644
index 000000000000..dd19a64314ca
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -0,0 +1,232 @@
+package com.google.gcloud.datastore;
+
+import com.google.api.client.util.Strings;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/**
+ * Utility to start and stop local Google Cloud Datastore process.
+ */
+public class LocalGcdHelper {
+
+  private final String dataset;
+  private Path gcdPath;
+  private ProcessStreamReader processReader;
+
+  public static final String DEFAULT_DATASET = "dataset1";
+  public static final int PORT = 8080;
+  private static final String GCD = "gcd-v1beta2-rev1-2.1.1";
+  private static final String GCD_LOC = "/" + GCD + ".zip";
+
+  private static class ProcessStreamReader extends Thread {
+
+    private final Process process;
+    private final BufferedReader reader;
+
+    ProcessStreamReader(Process process, String blockUntil) throws IOException {
+      super("Local GCD InputStream reader");
+      setDaemon(true);
+      this.process = process;
+      reader =  new BufferedReader(new InputStreamReader(process.getInputStream()));
+      if (!Strings.isNullOrEmpty(blockUntil)) {
+        String line;
+        do {
+          line = reader.readLine();
+        } while (line != null && !line.contains(blockUntil));
+      }
+    }
+
+    void terminate() throws InterruptedException, IOException {
+      process.destroy();
+      process.waitFor();
+      reader.close();
+    }
+
+    @Override
+    public void run() {
+      try {
+        while (reader.readLine() != null) {
+          // consume line
+        }
+      } catch (IOException e) {
+        // ignore
+      }
+    }
+
+    public static ProcessStreamReader start(Process process, String blockUntil) throws IOException {
+      ProcessStreamReader thread = new ProcessStreamReader(process, blockUntil);
+      thread.start();
+      return thread;
+    }
+  }
+
+  public LocalGcdHelper(String dataset) {
+    this.dataset = dataset;
+  }
+
+  public void start() throws IOException, InterruptedException {
+    sendQuitRequest();
+    gcdPath = Files.createTempDirectory("gcd");
+    File gcdFolder = gcdPath.toFile();
+    gcdFolder.deleteOnExit();
+
+    try (ZipInputStream zipIn = new ZipInputStream(getClass().getResourceAsStream(GCD_LOC))) {
+      ZipEntry entry = zipIn.getNextEntry();
+      while (entry != null) {
+        File filePath = new File(gcdFolder, entry.getName());
+        if (!entry.isDirectory()) {
+          extractFile(zipIn, filePath);
+        } else {
+          filePath.mkdir();
+        }
+        zipIn.closeEntry();
+        entry = zipIn.getNextEntry();
+      }
+    }
+
+    File datasetFolder = new File(gcdFolder, GCD + "/" + dataset);
+    datasetFolder.delete();
+
+    // TODO: When System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
+    Process temp = new ProcessBuilder()
+        .redirectErrorStream(true)
+        .directory(new File(gcdFolder, GCD))
+        .redirectOutput(new File("/dev/null"))
+        .command("sh", "gcd.sh", "create", "-d", dataset, dataset)
+        .start();
+    temp.waitFor();
+
+    temp = new ProcessBuilder()
+        .directory(new File(gcdFolder, GCD))
+        .redirectErrorStream(true)
+        .command("sh", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", dataset)
+        .start();
+    processReader = ProcessStreamReader.start(temp, "Dev App Server is now running");
+  }
+
+  private static void extractFile(ZipInputStream zipIn, File filePath) throws IOException {
+    try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) {
+      byte[] bytesIn = new byte[1024];
+      int read = 0;
+      while ((read = zipIn.read(bytesIn)) != -1) {
+        bos.write(bytesIn, 0, read);
+      }
+    }
+  }
+
+  public static void sendQuitRequest() {
+    try {
+      URL url = new URL("http", "localhost", PORT, "/_ah/admin/quit");
+      HttpURLConnection con = (HttpURLConnection) url.openConnection();
+      con.setRequestMethod("POST");
+      con.setDoOutput(true);
+      con.setDoInput(true);
+      OutputStream out = con.getOutputStream();
+      out.write("".getBytes());
+      out.flush();
+      InputStream in = con.getInputStream();
+      while (in.read() != -1) {
+        // consume input
+      }
+    } catch (IOException ignore) {
+      // ignore
+    }
+  }
+
+  public void stop() throws IOException, InterruptedException {
+    sendQuitRequest();
+    if (processReader != null) {
+      processReader.terminate();
+    }
+    if (gcdPath != null) {
+      deleteRecurse(gcdPath);
+      gcdPath = null;
+    }
+  }
+
+  private static void deleteRecurse(Path path) throws IOException {
+    if (path == null || !Files.exists(path)) {
+      return;
+    }
+    Files.walkFileTree(path, new SimpleFileVisitor() {
+
+      @Override
+      public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+        Files.delete(dir);
+        return FileVisitResult.CONTINUE;
+      }
+
+      @Override
+      public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+        Files.delete(file);
+        return FileVisitResult.CONTINUE;
+      }
+    });
+  }
+
+  public static LocalGcdHelper start(String dataset) throws IOException, InterruptedException {
+    LocalGcdHelper helper = new LocalGcdHelper(dataset);
+    helper.start();
+    return helper;
+  }
+
+  public static void main(String... args) throws IOException, InterruptedException {
+    if (args.length == 1) {
+      switch (args[0]) {
+        case "START":
+          if (!isActive(DEFAULT_DATASET)) {
+            LocalGcdHelper helper = LocalGcdHelper.start(DEFAULT_DATASET);
+            try (FileWriter writer = new FileWriter(".local_gcd_helper")) {
+              writer.write(helper.gcdPath.toAbsolutePath().toString());
+            }
+          }
+          return;
+        case "STOP":
+          sendQuitRequest();
+          File file = new File(".local_gcd_helper");
+          if (file.exists()) {
+            try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
+              String path = reader.readLine();
+              deleteRecurse(Paths.get(path));
+            }
+          }
+          file.delete();
+          return;
+        default:
+          break;
+      }
+    }
+    throw new RuntimeException("expeting only START | STOP");
+  }
+
+  public static boolean isActive(String dataset) {
+    try {
+      String path = "/datastore/v1beta2/datasets/" + dataset + "/lookup";
+      URL url = new URL("http://localhost:" + PORT + path);
+      try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
+        return "Valid RPC".equals(reader.readLine());
+      }
+    } catch (IOException ex) {
+      return false;
+    }
+  }
+}

From 14abf09f7b7bb7eb8d60481bdc676983779545b6 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 2 Jan 2015 18:03:48 -0800
Subject: [PATCH 076/732] minor change

---
 pom.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pom.xml b/pom.xml
index 5ed4ff0898f1..69c2673fe060 100644
--- a/pom.xml
+++ b/pom.xml
@@ -87,6 +87,10 @@
       http://repo.maven.apache.org/maven2
     
   
+  
+    UTF-8
+    UTF-8
+  
   
     
       

From bf4f0351afc8431891f5243a6d627359571e638d Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Fri, 2 Jan 2015 23:45:23 -0800
Subject: [PATCH 077/732] replace sh with bash

---
 src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index dd19a64314ca..8be2523de0f8 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -111,14 +111,14 @@ public void start() throws IOException, InterruptedException {
         .redirectErrorStream(true)
         .directory(new File(gcdFolder, GCD))
         .redirectOutput(new File("/dev/null"))
-        .command("sh", "gcd.sh", "create", "-d", dataset, dataset)
+        .command("bash", "gcd.sh", "create", "-d", dataset, dataset)
         .start();
     temp.waitFor();
 
     temp = new ProcessBuilder()
         .directory(new File(gcdFolder, GCD))
         .redirectErrorStream(true)
-        .command("sh", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", dataset)
+        .command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", dataset)
         .start();
     processReader = ProcessStreamReader.start(temp, "Dev App Server is now running");
   }

From e7ce4ead8e76dc06dfe9dd6f2aa9ea73ad9c3ccc Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 4 Jan 2015 21:55:37 -0800
Subject: [PATCH 078/732] cleanup

---
 RobustaSettings.xml                           |  2 +-
 pmd.xml                                       | 48 ++++++-------------
 .../java/com/google/gcloud/RetryHelper.java   | 19 ++++----
 .../com/google/gcloud/datastore/Cursor.java   |  4 +-
 .../datastore/DatastoreServiceOptions.java    |  2 +-
 .../gcloud/datastore/TransactionOption.java   |  9 ++--
 .../google/gcloud/datastore/Validator.java    |  5 +-
 .../gcloud/datastore/LocalGcdHelper.java      |  3 +-
 .../gcloud/datastore/SerializationTest.java   | 21 ++++----
 9 files changed, 51 insertions(+), 62 deletions(-)

diff --git a/RobustaSettings.xml b/RobustaSettings.xml
index e957a402fb2a..8c664c57feb7 100644
--- a/RobustaSettings.xml
+++ b/RobustaSettings.xml
@@ -1,2 +1,2 @@
 
-
+
diff --git a/pmd.xml b/pmd.xml
index 453c40c8784a..45b3e65a4cc6 100644
--- a/pmd.xml
+++ b/pmd.xml
@@ -6,22 +6,16 @@
    PMD Plugin preferences rule set
    
    
-   
-   
    
    
    
-   
-   
    
-   
    
    
    
    
    
    
-   
    
    
    
@@ -31,12 +25,9 @@
    
    
    
-   
-   
    
    
    
-   
    
    
    
@@ -59,7 +50,6 @@
    
    
    
-   
    
    
    
@@ -67,7 +57,6 @@
    
    
    
-   
    
    
    
@@ -80,19 +69,24 @@
    
    
    
-   
+   
+	   
+		   
+	   
+   
    
-   
    
    
    
    
    
-   
+   
+	   
+		   
+	   
+   
    
-   
    
-   
    
    
    
@@ -107,7 +101,6 @@
    
    
    
-   
    
    
    
@@ -149,10 +142,8 @@
    
    
    
-   
    
    
-   
    
    
    
@@ -171,25 +162,20 @@
    
    
    
-   
    
    
-   
    
    
    
    
-   
    
    
    
    
    
-   
    
    
    
-   
    
    
    
@@ -222,9 +208,7 @@
    
    
    
-   
    
-   
    
    
    
@@ -243,10 +227,13 @@
    
    
    
-   
+   
+	   
+		   
+	   
+   
    
    
-   
    
    
    
@@ -276,9 +263,6 @@
    
    
    
-   
-   
-   
    
    
    
@@ -312,7 +296,6 @@
    
    
    
-   
    
    
    
@@ -328,7 +311,6 @@
    
    
    
-   
    
    
 
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index ed0fcc1dab85..5e4445361227 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -18,9 +18,9 @@
 import java.util.logging.Logger;
 
 /**
- * Utility class for retrying operations. For more details about the parameters, see
- * {@link RetryParams}. If the request is never successful, a {@link RetriesExhaustedException} will
- * be thrown.
+ * Utility class for retrying operations.
+ * For more details about the parameters, see {@link RetryParams}.
+ * If the request is never successful, a {@link RetriesExhaustedException} will be thrown.
  *
  * @param  return value of the closure that is being run with retries
  */
@@ -99,8 +99,8 @@ public static final class NonRetriableException extends RetryHelperException {
 
     private static final long serialVersionUID = -2331878521983499652L;
 
-    NonRetriableException(Throwable ex) {
-      super(ex);
+    NonRetriableException(Throwable throwable) {
+      super(throwable);
     }
   }
 
@@ -164,12 +164,13 @@ private V doRetry() throws RetryHelperException {
           log.fine(this + ": attempt #" + attemptNumber + " succeeded");
         }
         return value;
+      } catch (InterruptedException | InterruptedIOException | ClosedByInterruptException e) {
+        if (!exceptionHandler.shouldRetry(e)) {
+          RetryInterruptedException.propagate();
+        }
+        exception = e;
       } catch (Exception e) {
         if (!exceptionHandler.shouldRetry(e)) {
-          if (e instanceof InterruptedException || e instanceof InterruptedIOException
-              || e instanceof ClosedByInterruptException) {
-            RetryInterruptedException.propagate();
-          }
           throw new NonRetriableException(e);
         }
         exception = e;
diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java
index e95f25144c12..bffdfd3dd9ac 100644
--- a/src/main/java/com/google/gcloud/datastore/Cursor.java
+++ b/src/main/java/com/google/gcloud/datastore/Cursor.java
@@ -65,7 +65,7 @@ public String toUrlSafe() {
     try {
       return URLEncoder.encode(toPb().toString(), UTF_8.name());
     } catch (UnsupportedEncodingException e) {
-      throw new RuntimeException("Unxpeced encoding exception", e);
+      throw new IllegalStateException("Unxpeced encoding exception", e);
     }
   }
 
@@ -77,7 +77,7 @@ public static Cursor fromUrlSafe(String urlSafe) {
       String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name());
       return fromPb(DatastoreV1.Value.parseFrom(utf8Str.getBytes()));
     } catch (UnsupportedEncodingException | InvalidProtocolBufferException e) {
-      throw new RuntimeException("Unxpeced decoding exception", e);
+      throw new IllegalStateException("Unxpeced decoding exception", e);
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index f3cc48d69801..009b663ab874 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -35,7 +35,7 @@ public static class Builder extends ServiceOptions.Builder {
 
     private String dataset;
     private String namespace;
-    private boolean force = false;
+    private boolean force;
     private Datastore datastore;
 
     private Builder() {
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
index 597715df20a8..9de8ea8ff745 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionOption.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
@@ -61,6 +61,11 @@ public IsolationLevel(Level level) {
     public Level level() {
       return level;
     }
+
+    @Override
+    BatchWriteOption toBatchWriteOption() {
+      return null;
+    }
   }
 
   TransactionOption() {
@@ -89,7 +94,5 @@ static Map, TransactionOption> asImmutableMap
     return builder.build();
   }
 
-  BatchWriteOption toBatchWriteOption() {
-    return null;
-  }
+  abstract BatchWriteOption toBatchWriteOption();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/src/main/java/com/google/gcloud/datastore/Validator.java
index 2f915ccc3d93..235c81b77f16 100644
--- a/src/main/java/com/google/gcloud/datastore/Validator.java
+++ b/src/main/java/com/google/gcloud/datastore/Validator.java
@@ -9,7 +9,7 @@
 /**
  * Utility to validate Datastore type/values.
  */
-class Validator {
+final class Validator {
 
   private static final Pattern DATASET_PATTERN = Pattern.compile(
       "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}\\:)?([a-z\\d][a-z\\d\\-]{0,99})");
@@ -17,6 +17,9 @@ class Validator {
   private static final Pattern NAMESPACE_PATTERN =
       Pattern.compile(String.format("[0-9A-Za-z\\._\\-]{0,%d}", MAX_NAMESPACE_LENGTH));
 
+  private Validator() {
+    // utility class
+  }
 
   static String validateDataset(String dataset) {
     checkArgument(!Strings.isNullOrEmpty(dataset), "dataset can't be empty or null");
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 8be2523de0f8..a4f1ed1ff01f 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -106,7 +106,7 @@ public void start() throws IOException, InterruptedException {
     File datasetFolder = new File(gcdFolder, GCD + "/" + dataset);
     datasetFolder.delete();
 
-    // TODO: When System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
+    // TODO: if System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
     Process temp = new ProcessBuilder()
         .redirectErrorStream(true)
         .directory(new File(gcdFolder, GCD))
@@ -159,7 +159,6 @@ public void stop() throws IOException, InterruptedException {
     }
     if (gcdPath != null) {
       deleteRecurse(gcdPath);
-      gcdPath = null;
     }
   }
 
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 636ab831c8ac..d87b64d7f480 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -17,6 +17,7 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 
@@ -97,7 +98,8 @@ public class SerializationTest {
   private static final ProjectionEntity PROJECTION_ENTITY = ProjectionEntity.fromPb(ENTITY1.toPb());
 
   @SuppressWarnings("rawtypes")
-  private Multimap typeToValues = ImmutableMultimap.builder()
+  private static final Multimap TYPE_TO_VALUES =
+      ImmutableMultimap.builder()
       .put(Type.NULL, NULL_VALUE)
       .put(Type.KEY, KEY_VALUE)
       .put(Type.STRING, STRING_VALUE)
@@ -115,7 +117,7 @@ public class SerializationTest {
   @Test
   public void testValues() throws Exception {
     for (Type type : Type.values()) {
-      for (Value value : typeToValues.get(type)) {
+      for (Value value : TYPE_TO_VALUES.get(type)) {
         Value copy = serialiazeAndDeserialize(value);
         assertEquals(value, value);
         assertEquals(value, copy);
@@ -141,15 +143,14 @@ public void testTypes() throws Exception {
   }
 
   @SuppressWarnings("unchecked")
-  private  T serialiazeAndDeserialize(T obj) throws Exception {
-    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-    try (ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream)) {
-      objectOutputStream.writeObject(obj);
+  private  T serialiazeAndDeserialize(T obj) throws IOException, ClassNotFoundException {
+    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+    try (ObjectOutputStream output = new ObjectOutputStream(bytes)) {
+      output.writeObject(obj);
     }
-    byte[] bytes = byteArrayOutputStream.toByteArray();
-    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes);
-    try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) {
-      return (T) in.readObject();
+    try (ObjectInputStream input =
+        new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
+      return (T) input.readObject();
     }
   }
 }

From 6019e60c7fe225b32a95d91848afbcc6dc22a09c Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Mon, 5 Jan 2015 13:23:29 -0800
Subject: [PATCH 079/732] more cleanup

---
 .checkstyle                                   |  6 ++--
 pmd.xml                                       | 30 +++++--------------
 .../com/google/gcloud/ExceptionHandler.java   |  2 +-
 .../google/gcloud/datastore/BatchWriter.java  |  4 +--
 .../gcloud/datastore/BatchWriterImpl.java     |  9 +++---
 .../datastore/DatastoreServiceException.java  |  2 +-
 .../com/google/gcloud/datastore/GqlQuery.java |  4 +--
 .../java/com/google/gcloud/datastore/Key.java |  8 ++---
 .../google/gcloud/datastore/PartialKey.java   |  2 +-
 .../google/gcloud/datastore/Serializable.java | 12 ++++----
 .../gcloud/datastore/StructuredQuery.java     |  8 ++---
 .../java/com/google/gcloud/storage/Acl.java   |  2 +-
 .../DatastoreServiceIntegrationTest.java      |  2 --
 13 files changed, 37 insertions(+), 54 deletions(-)

diff --git a/.checkstyle b/.checkstyle
index 5783bc0d77a1..2ad23d5136f6 100644
--- a/.checkstyle
+++ b/.checkstyle
@@ -1,7 +1,7 @@
 
 
-
-  
-    
+
+  
+    
   
 
diff --git a/pmd.xml b/pmd.xml
index 45b3e65a4cc6..62a87e6d063a 100644
--- a/pmd.xml
+++ b/pmd.xml
@@ -3,7 +3,8 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          name="pmd"
          xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd">
-   PMD Plugin preferences rule set
+	 PMD Plugin preferences rule set
+	 .*/test/.*
    
    
    
@@ -27,7 +28,6 @@
    
    
    
-   
    
    
    
@@ -69,23 +69,11 @@
    
    
    
-   
-	   
-		   
-	   
-   
    
    
    
    
    
-   
-   
-	   
-		   
-	   
-   
-   
    
    
    
@@ -103,6 +91,11 @@
    
    
    
+   
+	   
+		   
+	   
+   
    
    
    
@@ -127,7 +120,6 @@
    
    
    
-   
    
    
    
@@ -165,7 +157,6 @@
    
    
    
-   
    
    
    
@@ -185,10 +176,7 @@
    
    
    
-   
    
-   
-   
    
    
    
@@ -233,7 +221,6 @@
 	   
    
    
-   
    
    
    
@@ -243,7 +230,6 @@
    
    
    
-   
    
    
    
@@ -306,9 +292,7 @@
    
    
    
-   
    
-   
    
    
    
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index c3b558e58100..25a012ef061a 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -215,7 +215,7 @@ private static Method getCallableMethod(Class clazz) {
       return getCallableMethod(clazz.getSuperclass());
     } catch (SecurityException e) {
       // This should never happen
-      throw new RuntimeException("Unexpected exception", e);
+      throw new IllegalStateException("Unexpected exception", e);
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java
index 9fadd9eca6a9..d54bc0d51086 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java
@@ -45,7 +45,7 @@ public interface BatchWriter extends DatastoreWriter {
    * @throws DatastoreServiceException if batch is no longer active
    */
   @Override
-  public void delete(Key... key);
+  void delete(Key... key);
 
   /**
    * {@inheritDoc}
@@ -53,7 +53,7 @@ public interface BatchWriter extends DatastoreWriter {
    * @throws DatastoreServiceException if batch is no longer active
    */
   @Override
-  public void put(Entity... entity);
+  void put(Entity... entity);
 
   /**
    * Submit the batch to the Datastore.
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
index 3589fe3fec24..46713cc1d4a6 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
@@ -8,13 +8,14 @@
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.Map;
+import java.util.Set;
 
 class BatchWriterImpl implements BatchWriter {
 
-  private final LinkedHashMap toAdd = new LinkedHashMap<>();
-  private final LinkedHashMap toUpdate = new LinkedHashMap<>();
-  private final LinkedHashMap toPut = new LinkedHashMap<>();
-  private final LinkedHashSet toDelete = new LinkedHashSet<>();
+  private final Map toAdd = new LinkedHashMap<>();
+  private final Map toUpdate = new LinkedHashMap<>();
+  private final Map toPut = new LinkedHashMap<>();
+  private final Set toDelete = new LinkedHashSet<>();
   private final boolean force;
   protected final DatastoreServiceImpl datastore;
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index dfb3c56cd262..863a8cddb6e3 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -118,7 +118,7 @@ static DatastoreServiceException translateAndThrow(DatastoreException exception)
         JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
         reason = error.getString("reason");
         message = error.getString("message");
-      } catch (JSONException ex) {
+      } catch (JSONException ignore) {
         // ignore - will be converted to unknown
       }
     }
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 8569bae8c674..fdb20fdf8178 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -146,8 +146,8 @@ public static final class Builder {
     private String namespace;
     private String queryString;
     private boolean allowLiteral;
-    private Map namedBindings = new TreeMap<>();
-    private List positionalBindings = new LinkedList<>();
+    private final Map namedBindings = new TreeMap<>();
+    private final List positionalBindings = new LinkedList<>();
 
     Builder(Type type, String query) {
       this.type = checkNotNull(type);
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index 113802a8ba5b..551bdce23123 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -114,14 +114,14 @@ public String toUrlSafe() {
     try {
       return URLEncoder.encode(toString(), UTF_8.name());
     } catch (UnsupportedEncodingException e) {
-      throw new RuntimeException("Unxpeced encoding exception", e);
+      throw new IllegalStateException("Unxpeced encoding exception", e);
     }
   }
 
   /**
    * Create a {@code Key} given its URL safe encoded form.
    *
-   * @throws RuntimeException when decoding fails
+   * @throws IllegalArgumentException when decoding fails
    */
   public static Key fromUrlSafe(String urlSafe) {
     try {
@@ -129,9 +129,9 @@ public static Key fromUrlSafe(String urlSafe) {
       DatastoreV1.Key keyPb = DatastoreV1.Key.parseFrom(ByteString.copyFromUtf8(utf8Str));
       return fromPb(keyPb);
     } catch (UnsupportedEncodingException e) {
-      throw new RuntimeException("Unxpeced decoding exception", e);
+      throw new IllegalStateException("Unxpeced decoding exception", e);
     } catch (InvalidProtocolBufferException e) {
-      throw new RuntimeException("Could not parse key", e);
+      throw new IllegalArgumentException("Could not parse key", e);
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java
index e819aab8ec0b..fa28ed906e2f 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java
@@ -65,7 +65,7 @@ static PartialKey fromPb(DatastoreV1.Key keyPb) {
       }
     }
     List pathElementsPb = keyPb.getPathElementList();
-    Preconditions.checkArgument(pathElementsPb.size() > 0, "Path must not be empty");
+    Preconditions.checkArgument(!pathElementsPb.isEmpty(), "Path must not be empty");
     ImmutableList.Builder pathBuilder = ImmutableList.builder();
     for (DatastoreV1.Key.PathElement pathElementPb : pathElementsPb) {
       pathBuilder.add(PathElement.fromPb(pathElementPb));
diff --git a/src/main/java/com/google/gcloud/datastore/Serializable.java b/src/main/java/com/google/gcloud/datastore/Serializable.java
index c9f6b7984512..3770ecfccf7f 100644
--- a/src/main/java/com/google/gcloud/datastore/Serializable.java
+++ b/src/main/java/com/google/gcloud/datastore/Serializable.java
@@ -20,14 +20,14 @@ public String toString() {
     return toPb().toString();
   }
 
-  private void writeObject(ObjectOutputStream out) throws IOException {
-    out.defaultWriteObject();
-    out.writeObject(toPb().toByteArray());
+  private void writeObject(ObjectOutputStream output) throws IOException {
+    output.defaultWriteObject();
+    output.writeObject(toPb().toByteArray());
   }
 
-  private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
-    in.defaultReadObject();
-    bytesPb = (byte[]) in.readObject();
+  private void readObject(ObjectInputStream input) throws IOException, ClassNotFoundException {
+    input.defaultReadObject();
+    bytesPb = (byte[]) input.readObject();
   }
 
   protected Object readResolve() throws ObjectStreamException {
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 84d7d65e2c25..49657064f524 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -577,13 +577,13 @@ public static Projection first(String property) {
 
   static class BaseBuilder> {
 
-    private Type type;
+    private final Type type;
     private String namespace;
     private String kind;
-    private List projection = new LinkedList<>();
+    private final List projection = new LinkedList<>();
     private Filter filter;
-    private List groupBy = new LinkedList<>();
-    private List orderBy = new LinkedList<>();
+    private final List groupBy = new LinkedList<>();
+    private final List orderBy = new LinkedList<>();
     private Cursor startCursor;
     private Cursor endCursor;
     private int offset;
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 13bbe2bfb44d..a0ef42844b7d 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -2,7 +2,7 @@
 
 public interface Acl {
 
-  public class ProjectTeam {
+  class ProjectTeam {
       // ProjectNumber: The project number.
   //ProjectNumber string `json:"projectNumber,omitempty"`
 
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index de43c169c827..93f59598948b 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -154,7 +154,6 @@ public void testTransactionWithRead() {
       transaction.commit();
       fail("Expecting a failure");
     } catch (DatastoreServiceException expected) {
-      expected.printStackTrace();
       assertEquals(DatastoreServiceException.Code.ABORTED, expected.code());
     }
   }
@@ -181,7 +180,6 @@ public void testTransactionWithQuery() {
       transaction.commit();
       fail("Expecting a failure");
     } catch (DatastoreServiceException expected) {
-      expected.printStackTrace();
       assertEquals(DatastoreServiceException.Code.ABORTED, expected.code());
     }
   }

From ff2827e0d5ba996788e0aa67d8f83f4ac6eac477 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Mon, 5 Jan 2015 18:14:28 -0800
Subject: [PATCH 080/732] more cleanup

---
 .gitignore                                    |  1 +
 git-demo.iml                                  | 48 +++++++++++++++++++
 .../java/com/google/gcloud/RetryHelper.java   |  4 +-
 .../gcloud/datastore/BatchWriterImpl.java     |  2 +-
 .../com/google/gcloud/datastore/Blob.java     |  7 +--
 .../com/google/gcloud/datastore/Cursor.java   |  9 ++--
 .../gcloud/datastore/DatastoreService.java    | 14 +++---
 .../com/google/gcloud/datastore/DateTime.java |  5 +-
 .../com/google/gcloud/datastore/GqlQuery.java | 14 +++---
 .../java/com/google/gcloud/datastore/Key.java |  4 +-
 .../google/gcloud/datastore/ListValue.java    |  5 +-
 .../gcloud/datastore/PartialEntity.java       |  2 +-
 .../google/gcloud/datastore/QueryResult.java  |  2 +-
 .../gcloud/datastore/StructuredQuery.java     | 13 ++---
 .../gcloud/datastore/TransactionImpl.java     |  2 +-
 .../google/gcloud/datastore/Validator.java    |  2 +-
 .../com/google/gcloud/RetryHelperTest.java    |  3 +-
 .../DatastoreServiceIntegrationTest.java      |  2 +-
 .../gcloud/datastore/LocalGcdHelper.java      |  5 +-
 19 files changed, 88 insertions(+), 56 deletions(-)
 create mode 100644 git-demo.iml

diff --git a/.gitignore b/.gitignore
index 3ae51aeeae93..bb7802aa123c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 /target/
 .settings
+.idea
diff --git a/git-demo.iml b/git-demo.iml
new file mode 100644
index 000000000000..292e56683deb
--- /dev/null
+++ b/git-demo.iml
@@ -0,0 +1,48 @@
+
+
+  
+    
+    
+    
+      
+      
+      
+      
+      
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  
+
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index 5e4445361227..fd2bce4411f3 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -122,7 +122,7 @@ public int getAttemptNumber() {
   }
 
   @VisibleForTesting
-  static final void setContext(Context ctx) {
+  static void setContext(Context ctx) {
     if (ctx == null) {
       context.remove();
     } else {
@@ -130,7 +130,7 @@ static final void setContext(Context ctx) {
     }
   }
 
-  static final Context getContext() {
+  static Context getContext() {
     return context.get();
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
index 46713cc1d4a6..83cc6cb0808c 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
@@ -34,7 +34,7 @@ class BatchWriterImpl implements BatchWriter {
 
   protected void checkActive() {
     if (!active) {
-      throwInvalidRequest(getName() + " is no longer active");
+      throw throwInvalidRequest(getName() + " is no longer active");
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java
index f1fabe2f4899..b78dcc6ae922 100644
--- a/src/main/java/com/google/gcloud/datastore/Blob.java
+++ b/src/main/java/com/google/gcloud/datastore/Blob.java
@@ -60,10 +60,7 @@ public boolean equals(Object obj) {
     if (obj == this) {
       return true;
     }
-    if (!(obj instanceof Blob)) {
-      return false;
-    }
-    return byteString.equals(((Blob) obj).byteString);
+    return obj instanceof Blob && byteString.equals(((Blob) obj).byteString);
   }
 
   /**
@@ -113,7 +110,7 @@ public void copyTo(ByteBuffer target) {
   /**
    * Copies bytes into a buffer.
    *
-   * @throws java.io.IndexOutOfBoundsException if an offset or size is negative or too large
+   * @throws IndexOutOfBoundsException if an offset or size is negative or too large
    */
   public void copyTo(byte[] target) {
     byteString.copyTo(target, 0, 0, length());
diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java
index bffdfd3dd9ac..6f4a0b682ab0 100644
--- a/src/main/java/com/google/gcloud/datastore/Cursor.java
+++ b/src/main/java/com/google/gcloud/datastore/Cursor.java
@@ -38,10 +38,7 @@ public boolean equals(Object obj) {
     if (obj == this) {
       return true;
     }
-    if (!(obj instanceof Cursor)) {
-      return false;
-    }
-    return byteString.equals(((Cursor) obj).byteString);
+    return obj instanceof Cursor && byteString.equals(((Cursor) obj).byteString);
   }
 
   @Override
@@ -65,7 +62,7 @@ public String toUrlSafe() {
     try {
       return URLEncoder.encode(toPb().toString(), UTF_8.name());
     } catch (UnsupportedEncodingException e) {
-      throw new IllegalStateException("Unxpeced encoding exception", e);
+      throw new IllegalStateException("Unexpected encoding exception", e);
     }
   }
 
@@ -77,7 +74,7 @@ public static Cursor fromUrlSafe(String urlSafe) {
       String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name());
       return fromPb(DatastoreV1.Value.parseFrom(utf8Str.getBytes()));
     } catch (UnsupportedEncodingException | InvalidProtocolBufferException e) {
-      throw new IllegalStateException("Unxpeced decoding exception", e);
+      throw new IllegalStateException("Unexpected decoding exception", e);
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index f0f84191b18e..48ad05fe5713 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -15,7 +15,7 @@ public interface DatastoreService extends DatastoreReaderWriter {
   /**
    * Returns a new Datastore transaction.
    *
-   * @throws DatastoreServiceExcepiton upon failure
+   * @throws DatastoreServiceException upon failure
    */
   Transaction newTransaction(TransactionOption... options);
 
@@ -30,7 +30,7 @@ public interface DatastoreService extends DatastoreReaderWriter {
    * The returned key will have the same information (dataset, kind, namespace and ancestors)
    * as the given key and will have a newly assigned id.
    *
-   * @throws DatastoreServiceExcepiton upon failure
+   * @throws DatastoreServiceException upon failure
    */
   Key allocateId(PartialKey key);
 
@@ -38,34 +38,34 @@ public interface DatastoreService extends DatastoreReaderWriter {
    * Returns a list of keys using the allocated ids ordered by the input.
    *
    * @see #allocateId(PartialKey)
-   * @throws DatastoreServiceExcepiton upon failure
+   * @throws DatastoreServiceException upon failure
    */
   Iterator allocateId(PartialKey key, PartialKey... others);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceExcepiton upon failure
+   * @throws DatastoreServiceException upon failure
    */
   @Override
   void add(Entity... entity);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceExcepiton upon failure
+   * @throws DatastoreServiceException upon failure
    */
   @Override
   void update(Entity... entity);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceExcepiton upon failure
+   * @throws DatastoreServiceException upon failure
    */
   @Override
   void put(Entity... entity);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceExcepiton upon failure
+   * @throws DatastoreServiceException upon failure
    */
   @Override
   void delete(Key... key);
diff --git a/src/main/java/com/google/gcloud/datastore/DateTime.java b/src/main/java/com/google/gcloud/datastore/DateTime.java
index e04d603ff031..8c9ec1dbdca6 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTime.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTime.java
@@ -42,10 +42,7 @@ public boolean equals(Object obj) {
     if (obj == this) {
       return true;
     }
-    if (!(obj instanceof DateTime)) {
-      return false;
-    }
-    return timestampMicroseconds == ((DateTime) obj).timestampMicroseconds;
+    return obj instanceof DateTime && timestampMicroseconds == ((DateTime) obj).timestampMicroseconds;
   }
 
   public long timestampMicroseconds() {
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index fdb20fdf8178..72d3f8c8e5bc 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -27,9 +27,9 @@
  * 
A usage example:

  *
  * 
When the type of the results is known the preferred usage would be:
- * 
 {@code
- *   Query query = GqlQuery.builder(Query.Type.FULL, "select * from kind").build();
- *   QueryResult results = datastore.run(query);
+ * 
{@code
+ *   Query<Entity> query = GqlQuery.builder(Query.Type.FULL, "select * from kind").build();
+ *   QueryResult<Entity> results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -37,11 +37,11 @@
  * } 

  *
  * 
When the type of the results is unknown you can use this approach:
- * 
 {@code
- *   Query query = GqlQuery.builder("select __key__ from kind").build();
- *   QueryResult results = datastore.run(query);
+ * 
{@code
+ *   Query<?> query = GqlQuery.builder("select __key__ from kind").build();
+ *   QueryResult<?> results = datastore.run(query);
  *   if (Key.class.isAssignableFrom(results.resultClass())) {
- *     QueryResult keys = (QueryResult) results;
+ *     QueryResult<Key> keys = (QueryResult<Key>) results;
  *     while (keys.hasNext()) {
  *       Key key = keys.next();
  *       ...
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index 551bdce23123..c35efaeb6849 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -114,7 +114,7 @@ public String toUrlSafe() {
     try {
       return URLEncoder.encode(toString(), UTF_8.name());
     } catch (UnsupportedEncodingException e) {
-      throw new IllegalStateException("Unxpeced encoding exception", e);
+      throw new IllegalStateException("Unexpected encoding exception", e);
     }
   }
 
@@ -129,7 +129,7 @@ public static Key fromUrlSafe(String urlSafe) {
       DatastoreV1.Key keyPb = DatastoreV1.Key.parseFrom(ByteString.copyFromUtf8(utf8Str));
       return fromPb(keyPb);
     } catch (UnsupportedEncodingException e) {
-      throw new IllegalStateException("Unxpeced decoding exception", e);
+      throw new IllegalStateException("Unexpected decoding exception", e);
     } catch (InvalidProtocolBufferException e) {
       throw new IllegalArgumentException("Could not parse key", e);
     }
diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java
index 5af230d768b1..5b932c8c1da1 100644
--- a/src/main/java/com/google/gcloud/datastore/ListValue.java
+++ b/src/main/java/com/google/gcloud/datastore/ListValue.java
@@ -69,8 +69,7 @@ public Builder addValue(Value first, Value... other) {
     @Override
     public Builder indexed(boolean indexed) {
       // see b/18704917
-      DatastoreServiceException.throwInvalidRequest("ListValue can't specify index");
-      return this;
+      throw DatastoreServiceException.throwInvalidRequest("ListValue can't specify index");
     }
 
     /**
@@ -80,7 +79,7 @@ public Builder indexed(boolean indexed) {
      */
     @Override
     public Builder set(List> values) {
-      listBuilder = ImmutableList.>builder();
+      listBuilder = ImmutableList.builder();
       for (Value value : values) {
         addValue(value);
       }
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index df520d3f48b9..4f3a71955455 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -52,7 +52,7 @@ protected PartialEntity(PartialKey key, ImmutableSortedMap> pro
    * with the given {@code key}.
    */
   public Entity toEntity(Key key) {
-    return new Entity(key, ImmutableSortedMap.>copyOf(properties()));
+    return new Entity(key, ImmutableSortedMap.copyOf(properties()));
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index 3dab5967e81d..95fb993b1771 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -9,7 +9,7 @@
  * Results are loaded lazily therefore it is possible to get a {@code DatastoreServiceException}
  * upon {@link Iterator#hasNext hasNext} or {@link Iterator#next next} calls.
  *
- * @param V the type of the results value.
+ * @param  the type of the results value.
  */
 public interface QueryResult extends Iterator {
 
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 49657064f524..7cd197e27ee4 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -17,10 +17,7 @@
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.io.Serializable;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 
 /**
  * An implementation of a Google Cloud Datastore Query that can be constructed by providing
@@ -648,9 +645,7 @@ public B orderBy(OrderBy orderBy, OrderBy... others) {
 
     public B addOrderBy(OrderBy orderBy, OrderBy... others) {
       this.orderBy.add(orderBy);
-      for (OrderBy other : others) {
-        this.orderBy.add(other);
-      }
+      Collections.addAll(this.orderBy, others);
       return self();
     }
 
@@ -686,9 +681,7 @@ protected B groupBy(String property, String... others) {
 
     protected B addGroupBy(String property, String... others) {
       this.groupBy.add(property);
-      for (String other : others) {
-        this.groupBy.add(other);
-      }
+      Collections.addAll(this.groupBy, others);
       return self();
     }
 
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 8b43f421b9e5..0bd51b0e8615 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -90,7 +90,7 @@ protected String getName() {
   protected void checkActive() {
     super.checkActive();
     if (wasRolledback) {
-      throwInvalidRequest(getName() + " is not active (was rolledback)");
+      throw throwInvalidRequest(getName() + " is not active (was rolledback)");
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/src/main/java/com/google/gcloud/datastore/Validator.java
index 235c81b77f16..4d4afbc120b4 100644
--- a/src/main/java/com/google/gcloud/datastore/Validator.java
+++ b/src/main/java/com/google/gcloud/datastore/Validator.java
@@ -12,7 +12,7 @@
 final class Validator {
 
   private static final Pattern DATASET_PATTERN = Pattern.compile(
-      "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}\\:)?([a-z\\d][a-z\\d\\-]{0,99})");
+      "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}:)?([a-z\\d][a-z\\d\\-]{0,99})");
   private static final int MAX_NAMESPACE_LENGTH = 100;
   private static final Pattern NAMESPACE_PATTERN =
       Pattern.compile(String.format("[0-9A-Za-z\\._\\-]{0,%d}", MAX_NAMESPACE_LENGTH));
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/src/test/java/com/google/gcloud/RetryHelperTest.java
index 257dc892d229..1fafae35fd50 100644
--- a/src/test/java/com/google/gcloud/RetryHelperTest.java
+++ b/src/test/java/com/google/gcloud/RetryHelperTest.java
@@ -87,8 +87,7 @@ class E4 extends E2 {}
     try {
       RetryHelper.runWithRetries(new Callable() {
         @Override public Void call() throws E1 {
-          E1 exception = exceptions.next();
-          throw exception;
+          throw exceptions.next();
         }
       }, params, handler);
       fail("Exception should have been thrown");
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index 93f59598948b..1d943abf172d 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -525,7 +525,7 @@ public void testGetArray() {
     assertEquals(false, entity3.getBoolean("bool"));
     assertEquals(LIST_VALUE2.get(), entity3.getList("list"));
     PartialEntity partial1 = entity3.getEntity("partial1");
-    Entity partial2 = (Entity) entity3.getEntity("partial2");
+    Entity partial2 = entity3.getEntity("partial2");
     assertEquals(partial1, PARTIAL_ENTITY2);
     assertEquals(partial2, ENTITY2);
     assertEquals(Value.Type.BOOLEAN, entity3.getValue("bool").type());
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index a4f1ed1ff01f..519b7ae3ee72 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -126,7 +126,7 @@ public void start() throws IOException, InterruptedException {
   private static void extractFile(ZipInputStream zipIn, File filePath) throws IOException {
     try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) {
       byte[] bytesIn = new byte[1024];
-      int read = 0;
+      int read;
       while ((read = zipIn.read(bytesIn)) != -1) {
         bos.write(bytesIn, 0, read);
       }
@@ -214,9 +214,10 @@ public static void main(String... args) throws IOException, InterruptedException
           break;
       }
     }
-    throw new RuntimeException("expeting only START | STOP");
+    throw new RuntimeException("expecting only START | STOP");
   }
 
+  @SuppressWarnings("BooleanMethodIsAlwaysInverted")
   public static boolean isActive(String dataset) {
     try {
       String path = "/datastore/v1beta2/datasets/" + dataset + "/lookup";

From 7383b8959a5b59c57e8756553774b34138594ee2 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 6 Jan 2015 16:04:47 -0800
Subject: [PATCH 081/732] minor fix

---
 .gitignore                                    |   1 +
 git-demo.iml                                  | 201 +++++++++++++++++-
 .../java/com/google/gcloud/RetryHelper.java   |  13 +-
 3 files changed, 206 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index bb7802aa123c..3aeb1bf01c0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /target/
 .settings
 .idea
+*.iml
diff --git a/git-demo.iml b/git-demo.iml
index 292e56683deb..8f8bb491e93c 100644
--- a/git-demo.iml
+++ b/git-demo.iml
@@ -45,4 +45,203 @@
     
     
   
-
\ No newline at end of file
+  
+    
+    
+    
+  
+
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index fd2bce4411f3..4a590e64d0b3 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -1,12 +1,5 @@
 package com.google.gcloud;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.lang.Math.max;
-import static java.lang.Math.min;
-import static java.lang.Math.pow;
-import static java.lang.Math.random;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
@@ -17,6 +10,10 @@
 import java.util.concurrent.Callable;
 import java.util.logging.Logger;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.Math.*;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
 /**
  * Utility class for retrying operations.
  * For more details about the parameters, see {@link RetryParams}.
@@ -148,7 +145,7 @@ static Context getContext() {
   public String toString() {
     ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
     toStringHelper.add("stopwatch", stopwatch);
-    toStringHelper.add("attempNumber", attemptNumber);
+    toStringHelper.add("attemptNumber", attemptNumber);
     toStringHelper.add("callable", callable);
     return toStringHelper.toString();
   }

From 941c63e4072c56b4a8614f74481b6af477606960 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 6 Jan 2015 16:13:42 -0800
Subject: [PATCH 082/732] revert wrong formatting

---
 src/main/java/com/google/gcloud/RetryHelper.java | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index 4a590e64d0b3..25f58b4b8b79 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -1,5 +1,9 @@
 package com.google.gcloud;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.Math.*;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
@@ -10,10 +14,6 @@
 import java.util.concurrent.Callable;
 import java.util.logging.Logger;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.lang.Math.*;
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-
 /**
  * Utility class for retrying operations.
  * For more details about the parameters, see {@link RetryParams}.
@@ -157,7 +157,7 @@ private V doRetry() throws RetryHelperException {
       Exception exception;
       try {
         V value = callable.call();
-        if (attemptNumber > 1) {
+      if (attemptNumber > 1) {
           log.fine(this + ": attempt #" + attemptNumber + " succeeded");
         }
         return value;

From d3a014d9ac4a19ea1308ce33646fe162d1edb1c3 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 6 Jan 2015 16:17:31 -0800
Subject: [PATCH 083/732] revert wrong formatting

---
 src/main/java/com/google/gcloud/RetryHelper.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index 25f58b4b8b79..715141a01241 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -1,7 +1,10 @@
 package com.google.gcloud;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static java.lang.Math.*;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.lang.Math.pow;
+import static java.lang.Math.random;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.annotations.VisibleForTesting;

From 1178bb0ffd534c2784464725c79b9a579d9a5930 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 7 Jan 2015 22:54:17 -0800
Subject: [PATCH 084/732] even more cleanup

---
 .gitignore                                    |   1 +
 findbugs-exclude.xml                          |   2 +
 git-demo.iml                                  | 201 +++++++++++++++++-
 .../java/com/google/gcloud/AuthConfig.java    |   8 +-
 .../com/google/gcloud/ExceptionHandler.java   |  37 ++--
 .../java/com/google/gcloud/RetryHelper.java   |  16 +-
 .../com/google/gcloud/ServiceOptions.java     |  25 ++-
 .../google/gcloud/datastore/BaseEntity.java   |  38 ++--
 .../com/google/gcloud/datastore/BaseKey.java  |  20 +-
 .../gcloud/datastore/BatchWriterImpl.java     |  18 +-
 .../com/google/gcloud/datastore/Blob.java     |   7 +-
 .../com/google/gcloud/datastore/Cursor.java   |   7 +-
 .../datastore/DatastoreServiceImpl.java       |  24 ++-
 .../datastore/DatastoreServiceOptions.java    |   7 +-
 .../com/google/gcloud/datastore/DateTime.java |   6 +-
 .../com/google/gcloud/datastore/GqlQuery.java |  11 +-
 .../google/gcloud/datastore/PathElement.java  |   9 +-
 .../gcloud/datastore/ProjectionEntity.java    |   6 +-
 .../com/google/gcloud/datastore/Query.java    |   7 +-
 .../gcloud/datastore/QueryResultImpl.java     |   3 +-
 .../gcloud/datastore/StructuredQuery.java     |  38 ++--
 .../gcloud/datastore/TransactionImpl.java     |  10 +-
 .../google/gcloud/datastore/Validator.java    |   8 +-
 .../com/google/gcloud/datastore/Value.java    |   2 +-
 .../gcloud/storage/StorageServiceOptions.java |  22 +-
 .../google/gcloud/ExceptionHandlerTest.java   |  16 +-
 .../com/google/gcloud/RetryHelperTest.java    |  61 +++---
 .../DatastoreServiceIntegrationTest.java      |  12 +-
 .../gcloud/datastore/LocalGcdHelper.java      |  17 +-
 .../gcloud/datastore/SerializationTest.java   |  10 +-
 30 files changed, 439 insertions(+), 210 deletions(-)
 create mode 100644 findbugs-exclude.xml

diff --git a/.gitignore b/.gitignore
index bb7802aa123c..1188335c97d3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /target/
 .settings
 .idea
+.DS_Store
diff --git a/findbugs-exclude.xml b/findbugs-exclude.xml
new file mode 100644
index 000000000000..43bc3321deb5
--- /dev/null
+++ b/findbugs-exclude.xml
@@ -0,0 +1,2 @@
+
+
diff --git a/git-demo.iml b/git-demo.iml
index 292e56683deb..b5c10cc72045 100644
--- a/git-demo.iml
+++ b/git-demo.iml
@@ -10,7 +10,7 @@
       
       
     
-    
+    
     
     
     
@@ -45,4 +45,203 @@
     
     
   
+  
+    
+    
+    
+  
 
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java
index 67e7b572c133..e1c5646c66d9 100644
--- a/src/main/java/com/google/gcloud/AuthConfig.java
+++ b/src/main/java/com/google/gcloud/AuthConfig.java
@@ -57,9 +57,8 @@ protected HttpRequestInitializer httpRequestInitializer(
     }
   }
 
-  protected abstract HttpRequestInitializer httpRequestInitializer(
-      HttpTransport transport, Set scopes);
-
+  protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
+      Set scopes);
 
   public static AuthConfig createForAppEngine() {
     return new AppEngineAuthConfig();
@@ -69,7 +68,8 @@ public static AuthConfig createForComputeEngine() throws IOException, GeneralSec
     final ComputeCredential cred = getComputeCredential();
     return new AuthConfig() {
       @Override
-      protected HttpRequestInitializer httpRequestInitializer(HttpTransport ts, Set sc) {
+      protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
+          Set scopes) {
         return cred;
       }
     };
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index 25a012ef061a..7330a84ba06d 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -27,7 +27,7 @@ public final class ExceptionHandler implements Serializable {
   private final ImmutableList interceptors;
   private final ImmutableSet> retriableExceptions;
   private final ImmutableSet> nonRetriableExceptions;
-  private final Set retryInfos = Sets.newHashSet();
+  private final Set retryInfo = Sets.newHashSet();
 
   public interface Interceptor extends Serializable {
 
@@ -38,7 +38,7 @@ enum RetryResult {
 
       private final boolean booleanValue;
 
-      private RetryResult(boolean booleanValue) {
+      RetryResult(boolean booleanValue) {
         this.booleanValue = booleanValue;
       }
 
@@ -55,7 +55,7 @@ boolean booleanValue() {
      *     ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}),
      *     or evaluation should proceed ({@code null}).
      */
-    RetryResult shouldRetry(Exception exception);
+    RetryResult beforeEval(Exception exception);
 
     /**
      * This method is called after the evaluation and could alter its result.
@@ -66,7 +66,7 @@ boolean booleanValue() {
      *     ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}),
      *     or evaluation should proceed ({@code null}).
      */
-    RetryResult shouldRetry(Exception exception, RetryResult retryResult);
+    RetryResult afterEval(Exception exception, RetryResult retryResult);
   }
 
   /**
@@ -173,31 +173,30 @@ private ExceptionHandler(Builder builder) {
         Sets.intersection(retriableExceptions, nonRetriableExceptions).isEmpty(),
         "Same exception was found in both retriable and non-retriable sets");
     for (Class exception : retriableExceptions) {
-      addToRetryInfos(retryInfos, new RetryInfo(exception, Interceptor.RetryResult.RETRY));
+      addRetryInfo(new RetryInfo(exception, Interceptor.RetryResult.RETRY), retryInfo);
     }
     for (Class exception : nonRetriableExceptions) {
-      addToRetryInfos(retryInfos,  new RetryInfo(exception, Interceptor.RetryResult.ABORT));
+      addRetryInfo(new RetryInfo(exception, Interceptor.RetryResult.ABORT), retryInfo);
     }
   }
 
-  private static void addToRetryInfos(Set retryInfos, RetryInfo retryInfo) {
-    for (RetryInfo current : retryInfos) {
+  private static void addRetryInfo(RetryInfo retryInfo, Set dest) {
+    for (RetryInfo current : dest) {
       if (current.exception.isAssignableFrom(retryInfo.exception)) {
-        addToRetryInfos(current.children, retryInfo);
+        addRetryInfo(retryInfo, current.children);
         return;
       }
       if (retryInfo.exception.isAssignableFrom(current.exception)) {
         retryInfo.children.add(current);
       }
     }
-    retryInfos.removeAll(retryInfo.children);
-    retryInfos.add(retryInfo);
+    dest.removeAll(retryInfo.children);
+    dest.add(retryInfo);
   }
 
-
-  private static RetryInfo findMostSpecificRetryInfo(Set retryInfos,
+  private static RetryInfo findMostSpecificRetryInfo(Set retryInfo,
       Class exception) {
-    for (RetryInfo current : retryInfos) {
+    for (RetryInfo current : retryInfo) {
       if (current.exception.isAssignableFrom(exception)) {
         RetryInfo  match = findMostSpecificRetryInfo(current.children, exception);
         return match == null ? current : match;
@@ -223,10 +222,10 @@ void verifyCaller(Callable callable) {
     Method callMethod = getCallableMethod(callable.getClass());
     for (Class exceptionOrError : callMethod.getExceptionTypes()) {
       Preconditions.checkArgument(Exception.class.isAssignableFrom(exceptionOrError),
-          "Callable method exceptions must be dervied from Exception");
+          "Callable method exceptions must be derived from Exception");
       @SuppressWarnings("unchecked") Class exception =
           (Class) exceptionOrError;
-      Preconditions.checkArgument(findMostSpecificRetryInfo(retryInfos, exception) != null,
+      Preconditions.checkArgument(findMostSpecificRetryInfo(retryInfo, exception) != null,
           "Declared exception '" + exception + "' is not covered by exception handler");
     }
   }
@@ -241,16 +240,16 @@ public Set> getNonRetriableExceptions() {
 
   boolean shouldRetry(Exception ex) {
     for (Interceptor interceptor : interceptors) {
-      Interceptor.RetryResult retryResult = interceptor.shouldRetry(ex);
+      Interceptor.RetryResult retryResult = interceptor.beforeEval(ex);
       if (retryResult != null) {
         return retryResult.booleanValue();
       }
     }
-    RetryInfo retryInfo = findMostSpecificRetryInfo(retryInfos, ex.getClass());
+    RetryInfo retryInfo = findMostSpecificRetryInfo(this.retryInfo, ex.getClass());
     Interceptor.RetryResult retryResult =
         retryInfo == null ? Interceptor.RetryResult.ABORT : retryInfo.retry;
     for (Interceptor interceptor : interceptors) {
-      retryResult = firstNonNull(interceptor.shouldRetry(ex, retryResult), retryResult);
+      retryResult = firstNonNull(interceptor.afterEval(ex, retryResult), retryResult);
     }
     return retryResult.booleanValue();
   }
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index fd2bce4411f3..3c6e23cf085c 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -15,6 +15,7 @@
 import java.io.InterruptedIOException;
 import java.nio.channels.ClosedByInterruptException;
 import java.util.concurrent.Callable;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
@@ -67,7 +68,7 @@ public static final class RetryInterruptedException extends RetryHelperException
     RetryInterruptedException() {}
 
     /**
-     * Sets the caller thread interrupt flag and throws {@code RetryInteruptedException}.
+     * Sets the caller thread interrupt flag and throws {@code RetryInterruptedException}.
      */
     public static void propagate() throws RetryInterruptedException {
       Thread.currentThread().interrupt();
@@ -147,9 +148,11 @@ static Context getContext() {
   @Override
   public String toString() {
     ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+    toStringHelper.add("params", params);
     toStringHelper.add("stopwatch", stopwatch);
-    toStringHelper.add("attempNumber", attemptNumber);
+    toStringHelper.add("attemptNumber", attemptNumber);
     toStringHelper.add("callable", callable);
+    toStringHelper.add("exceptionHandler", exceptionHandler);
     return toStringHelper.toString();
   }
 
@@ -160,7 +163,7 @@ private V doRetry() throws RetryHelperException {
       Exception exception;
       try {
         V value = callable.call();
-        if (attemptNumber > 1) {
+        if (attemptNumber > 1 && log.isLoggable(Level.FINE)) {
           log.fine(this + ": attempt #" + attemptNumber + " succeeded");
         }
         return value;
@@ -181,11 +184,14 @@ private V doRetry() throws RetryHelperException {
         throw new RetriesExhaustedException(this + ": Too many failures, giving up", exception);
       }
       long sleepDurationMillis = getSleepDuration(params, attemptNumber);
-      log.fine(this + ": Attempt #" + attemptNumber + " failed [" + exception + "], sleeping for "
-          + sleepDurationMillis + " ms");
+      if (log.isLoggable(Level.FINE)) {
+        log.fine(this + ": Attempt #" + attemptNumber + " failed [" + exception + "], sleeping for "
+            + sleepDurationMillis + " ms");
+      }
       try {
         Thread.sleep(sleepDurationMillis);
       } catch (InterruptedException e) {
+        // propagate as RetryInterruptedException
         RetryInterruptedException.propagate();
       }
     }
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index c23d754a5c46..b34875f5f87f 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -2,6 +2,7 @@
 
 
 import static com.google.common.base.MoreObjects.firstNonNull;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
 import com.google.api.client.http.HttpRequestInitializer;
@@ -11,6 +12,7 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.Set;
@@ -121,10 +123,29 @@ protected static String googleCloudProjectId() {
       URLConnection connection = url.openConnection();
       connection.setRequestProperty("X-Google-Metadata-Request", "True");
       try (BufferedReader reader =
-          new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
+               new BufferedReader(new InputStreamReader(connection.getInputStream(), UTF_8))) {
         return reader.readLine();
       }
-    } catch (IOException e) {
+    } catch (IOException ignore) {
+      // return null if can't determine
+      return null;
+    }
+  }
+
+  protected static String getAppEngineProjectId() {
+    // TODO(ozarov): An alternative to reflection would be to depend on AE api jar:
+    // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0
+    try {
+      Class factoryClass =
+          Class.forName("com.google.appengine.api.appidentity.AppIdentityServiceFactory");
+      Method method = factoryClass.getMethod("getAppIdentityService");
+      Object appIdentityService = method.invoke(null);
+      method = appIdentityService.getClass().getMethod("getServiceAccountName");
+      String serviceAccountName = (String) method.invoke(appIdentityService);
+      int indexOfAtSign = serviceAccountName.indexOf('@');
+      return serviceAccountName.substring(0, indexOfAtSign);
+    } catch (Exception ignore) {
+      // return null if can't determine
       return null;
     }
   }
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 91665fd59987..a760d98fa90e 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -28,20 +28,20 @@ abstract class BaseEntity extends Serializable {
 
   private final transient ImmutableSortedMap> properties;
 
-  protected abstract static class Builder> {
+  abstract static class Builder> {
 
-    protected final Map> properties;
+    final Map> properties;
 
-    protected Builder() {
+    Builder() {
       properties = new HashMap<>();
     }
 
-    protected Builder(BaseEntity entity) {
+    Builder(BaseEntity entity) {
       properties = new HashMap<>(entity.properties());
     }
 
     @SuppressWarnings("unchecked")
-    protected B self() {
+    B self() {
       return (B) this;
     }
 
@@ -124,7 +124,7 @@ public B setNull(String name) {
     public abstract BaseEntity build();
   }
 
-  protected BaseEntity(ImmutableSortedMap> properties) {
+  BaseEntity(ImmutableSortedMap> properties) {
     this.properties = properties;
   }
 
@@ -153,41 +153,49 @@ public boolean isNull(String name) {
     return getValue(name) instanceof NullValue;
   }
 
+  @SuppressWarnings("unchecked")
   public String getString(String name) {
-    return ((StringValue) getValue(name)).get();
+    return ((Value) getValue(name)).get();
   }
 
+  @SuppressWarnings("unchecked")
   public long getLong(String name) {
-    return ((LongValue) getValue(name)).get();
+    return ((Value) getValue(name)).get();
   }
 
+  @SuppressWarnings("unchecked")
   public double getDouble(String name) {
-    return ((DoubleValue) getValue(name)).get();
+    return ((Value) getValue(name)).get();
   }
 
+  @SuppressWarnings("unchecked")
   public boolean getBoolean(String name) {
-    return ((BooleanValue) getValue(name)).get();
+    return ((Value) getValue(name)).get();
   }
 
+  @SuppressWarnings("unchecked")
   public DateTime getDateTime(String name) {
-    return ((DateTimeValue) getValue(name)).get();
+    return ((Value) getValue(name)).get();
   }
 
+  @SuppressWarnings("unchecked")
   public Key getKey(String name) {
-    return ((KeyValue) getValue(name)).get();
+    return ((Value) getValue(name)).get();
   }
 
   @SuppressWarnings("unchecked")
   public  T getEntity(String name) {
-    return (T) ((EntityValue) getValue(name)).get();
+    return (T) ((Value) getValue(name)).get();
   }
 
+  @SuppressWarnings("unchecked")
   public List> getList(String name) {
-    return ((ListValue) getValue(name)).get();
+    return ((Value>>) getValue(name)).get();
   }
 
+  @SuppressWarnings("unchecked")
   public Blob getBlob(String name) {
-    return ((BlobValue) getValue(name)).get();
+    return ((Value) getValue(name)).get();
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index ea9f600aff7e..534f390ca67c 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -25,24 +25,24 @@ abstract class BaseKey extends Serializable {
 
   abstract static class Builder> {
 
-    protected String dataset;
-    protected String namespace;
-    protected String kind;
-    protected final List ancestors;
+    String dataset;
+    String namespace;
+    String kind;
+    final List ancestors;
 
     private static final int MAX_PATH = 100;
 
-    public Builder(String dataset) {
+    Builder(String dataset) {
       this.dataset = validateDataset(dataset);
       ancestors = new LinkedList<>();
     }
 
-    public Builder(String dataset, String kind) {
+    Builder(String dataset, String kind) {
       this(dataset);
       this.kind = validateKind(kind);
     }
 
-    public Builder(BaseKey copyFrom) {
+    Builder(BaseKey copyFrom) {
       dataset = copyFrom.dataset();
       namespace = copyFrom.namespace();
       ancestors = new LinkedList<>(copyFrom.ancestors());
@@ -50,7 +50,7 @@ public Builder(BaseKey copyFrom) {
     }
 
     @SuppressWarnings("unchecked")
-    protected B self() {
+    B self() {
       return (B) this;
     }
 
@@ -120,11 +120,11 @@ public List ancestors() {
   /**
    * Returns an immutable list of the key's path (ancestors + self).
    */
-  public List path() {
+  List path() {
     return path;
   }
 
-  protected PathElement leaf() {
+  PathElement leaf() {
     return path().get(path().size() - 1);
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
index 83cc6cb0808c..273d46f254a3 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
@@ -17,7 +17,7 @@ class BatchWriterImpl implements BatchWriter {
   private final Map toPut = new LinkedHashMap<>();
   private final Set toDelete = new LinkedHashSet<>();
   private final boolean force;
-  protected final DatastoreServiceImpl datastore;
+  final DatastoreServiceImpl datastore;
 
   private boolean active = true;
 
@@ -32,19 +32,19 @@ class BatchWriterImpl implements BatchWriter {
     }
   }
 
-  protected void checkActive() {
+  void validateActive() {
     if (!active) {
       throw throwInvalidRequest(getName() + " is no longer active");
     }
   }
 
-  protected String getName() {
+  String getName() {
     return "batch";
   }
 
   @Override
   public void add(Entity... entities) {
-    checkActive();
+    validateActive();
     for (Entity entity : entities) {
       Key key = entity.key();
       if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) {
@@ -61,7 +61,7 @@ public void add(Entity... entities) {
 
   @Override
   public void update(Entity... entities) {
-    checkActive();
+    validateActive();
     for (Entity entity : entities) {
       Key key = entity.key();
       if (toDelete.contains(key)) {
@@ -78,7 +78,7 @@ public void update(Entity... entities) {
 
   @Override
   public void put(Entity... entities) {
-    checkActive();
+    validateActive();
     for (Entity entity : entities) {
       Key key = entity.key();
       toAdd.remove(key);
@@ -90,7 +90,7 @@ public void put(Entity... entities) {
 
   @Override
   public void delete(Key... keys) {
-    checkActive();
+    validateActive();
     for (Key key : keys) {
       toAdd.remove(key);
       toUpdate.remove(key);
@@ -101,7 +101,7 @@ public void delete(Key... keys) {
 
   @Override
   public void submit() {
-    checkActive();
+    validateActive();
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
     for (Entity entity : toAdd.values()) {
       mutationPb.addInsert(entity.toPb());
@@ -129,7 +129,7 @@ public boolean active() {
     return active;
   }
 
-  protected DatastoreV1.CommitRequest.Builder newCommitRequest() {
+  DatastoreV1.CommitRequest.Builder newCommitRequest() {
     DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
     requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL);
     return requestPb;
diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java
index b78dcc6ae922..b8d6892d201f 100644
--- a/src/main/java/com/google/gcloud/datastore/Blob.java
+++ b/src/main/java/com/google/gcloud/datastore/Blob.java
@@ -57,10 +57,7 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-    return obj instanceof Blob && byteString.equals(((Blob) obj).byteString);
+    return obj == this || obj instanceof Blob && byteString.equals(((Blob) obj).byteString);
   }
 
   /**
@@ -90,7 +87,7 @@ public ByteBuffer asReadOnlyByteBuffer() {
   public InputStream asInputStream() {
     final ByteBuffer byteBuffer = asReadOnlyByteBuffer();
     return new InputStream() {
-      @Override public int read() throws IOException {
+      @Override public int read() {
         return !byteBuffer.hasRemaining() ? -1 : byteBuffer.get() & 0xFF;
       }
     };
diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java
index 6f4a0b682ab0..25b35f6bcfe6 100644
--- a/src/main/java/com/google/gcloud/datastore/Cursor.java
+++ b/src/main/java/com/google/gcloud/datastore/Cursor.java
@@ -35,10 +35,7 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-    return obj instanceof Cursor && byteString.equals(((Cursor) obj).byteString);
+    return obj == this || obj instanceof Cursor && byteString.equals(((Cursor) obj).byteString);
   }
 
   @Override
@@ -72,7 +69,7 @@ public String toUrlSafe() {
   public static Cursor fromUrlSafe(String urlSafe) {
     try {
       String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name());
-      return fromPb(DatastoreV1.Value.parseFrom(utf8Str.getBytes()));
+      return fromPb(DatastoreV1.Value.parseFrom(ByteString.copyFromUtf8(utf8Str)));
     } catch (UnsupportedEncodingException | InvalidProtocolBufferException e) {
       throw new IllegalStateException("Unexpected decoding exception", e);
     }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index a887c03076df..024975b8cd03 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -15,6 +15,8 @@
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.Callable;
 
 
@@ -28,12 +30,12 @@ final class DatastoreServiceImpl implements DatastoreService {
         private static final long serialVersionUID = 6911242958397733203L;
 
         @Override
-        public RetryResult shouldRetry(Exception exception, RetryResult retryResult) {
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
           return null;
         }
 
         @Override
-        public RetryResult shouldRetry(Exception exception) {
+        public RetryResult beforeEval(Exception exception) {
           if (exception instanceof DatastoreServiceException) {
             boolean isTransient = ((DatastoreServiceException) exception).code().isTransient();
             return isTransient
@@ -63,13 +65,13 @@ public DatastoreServiceOptions options() {
   }
 
   @Override
-  public BatchWriter newBatchWriter(BatchWriteOption... batchWriteOption) {
-    return new BatchWriterImpl(this, batchWriteOption);
+  public BatchWriter newBatchWriter(BatchWriteOption... options) {
+    return new BatchWriterImpl(this, options);
   }
 
   @Override
-  public Transaction newTransaction(TransactionOption... transactionOption) {
-    return new TransactionImpl(this, transactionOption);
+  public Transaction newTransaction(TransactionOption... options) {
+    return new TransactionImpl(this, options);
   }
 
   @Override
@@ -209,9 +211,9 @@ DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) {
   }
 
   @Override
-  public void add(Entity... entities) {
+    public void add(Entity... entities) {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    LinkedHashSet keys = new LinkedHashSet<>();
+    Set keys = new LinkedHashSet<>();
     for (Entity entity : entities) {
       if (!keys.add(entity.key())) {
         throw DatastoreServiceException.throwInvalidRequest(
@@ -225,7 +227,7 @@ public void add(Entity... entities) {
   @Override
   public void update(Entity... entities) {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    LinkedHashMap dedupEntities = new LinkedHashMap<>();
+    Map dedupEntities = new LinkedHashMap<>();
     for (Entity entity : entities) {
       dedupEntities.put(entity.key(), entity);
     }
@@ -238,7 +240,7 @@ public void update(Entity... entities) {
   @Override
   public void put(Entity... entities) {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    LinkedHashMap dedupEntities = new LinkedHashMap<>();
+    Map dedupEntities = new LinkedHashMap<>();
     for (Entity entity : entities) {
       dedupEntities.put(entity.key(), entity);
     }
@@ -251,7 +253,7 @@ public void put(Entity... entities) {
   @Override
   public void delete(Key... keys) {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    LinkedHashSet dedupKeys = new LinkedHashSet<>(Arrays.asList(keys));
+    Set dedupKeys = new LinkedHashSet<>(Arrays.asList(keys));
     for (Key key : dedupKeys) {
       mutationPb.addDelete(key.toPb());
     }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 009b663ab874..262d8f81060f 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -73,7 +73,7 @@ public Builder force(boolean force) {
     }
   }
 
-  DatastoreServiceOptions(Builder builder) {
+  private DatastoreServiceOptions(Builder builder) {
     super(builder);
     namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
     force = builder.force;
@@ -140,8 +140,9 @@ private static String defaultNamespace() {
       Class clazz = Class.forName("com.google.appengine.api.NamespaceManager");
       Method method = clazz.getMethod("get");
       String namespace = (String) method.invoke(null);
-      return "".equals(namespace) ? null : namespace;
-    } catch (Exception ex) {
+      return namespace == null || namespace.isEmpty() ? null : namespace;
+    } catch (Exception ignore) {
+      // return null (Datastore default namespace) if could not automatically determine
       return null;
     }
   }
diff --git a/src/main/java/com/google/gcloud/datastore/DateTime.java b/src/main/java/com/google/gcloud/datastore/DateTime.java
index 8c9ec1dbdca6..a2e8c6398351 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTime.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTime.java
@@ -39,10 +39,8 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-    return obj instanceof DateTime && timestampMicroseconds == ((DateTime) obj).timestampMicroseconds;
+    return obj == this || obj instanceof DateTime
+        && timestampMicroseconds == ((DateTime) obj).timestampMicroseconds;
   }
 
   public long timestampMicroseconds() {
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 72d3f8c8e5bc..9c5c982b82c4 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -378,15 +378,14 @@ protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
   }
 
   @Override
-  protected Object fromPb(Type resultType, String namespace, byte[] bytesPb)
+  protected Object fromPb(Type type, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
-    return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
+    return fromPb(type, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
   }
 
-  static  GqlQuery fromPb(Type resultType, String namespace,
-      DatastoreV1.GqlQuery queryPb) {
-    Builder builder = new Builder<>(resultType, queryPb.getQueryString());
-    builder.namespace(namespace);
+  private static  GqlQuery fromPb(Type type, String ns, DatastoreV1.GqlQuery queryPb) {
+    Builder builder = new Builder<>(type, queryPb.getQueryString());
+    builder.namespace(ns);
     if (queryPb.hasAllowLiteral()) {
       builder.allowLiteral = queryPb.getAllowLiteral();
     }
diff --git a/src/main/java/com/google/gcloud/datastore/PathElement.java b/src/main/java/com/google/gcloud/datastore/PathElement.java
index 9e84764e55f7..b3ec8b55e5d3 100644
--- a/src/main/java/com/google/gcloud/datastore/PathElement.java
+++ b/src/main/java/com/google/gcloud/datastore/PathElement.java
@@ -89,11 +89,12 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   static PathElement fromPb(DatastoreV1.Key.PathElement pathElementPb) {
     String kind = pathElementPb.getKind();
     if (pathElementPb.hasId()) {
-      return PathElement.of(kind, pathElementPb.getId());
-    } else if (pathElementPb.hasName()) {
-      return PathElement.of(kind, pathElementPb.getName());
+      return of(kind, pathElementPb.getId());
     }
-    return PathElement.of(kind);
+    if (pathElementPb.hasName()) {
+      return of(kind, pathElementPb.getName());
+    }
+    return of(kind);
   }
 
   static PathElement of(String kind) {
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
index 6b219ce0968c..22201f82dda4 100644
--- a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -78,22 +78,24 @@ public Key key() {
     return key;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public DateTime getDateTime(String name) {
     Value value = getValue(name);
     if (value.hasMeaning() && value.meaning() == 18 && value instanceof LongValue) {
       return new DateTime(getLong(name));
     }
-    return ((DateTimeValue) value).get();
+    return ((Value) value).get();
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public Blob getBlob(String name) {
     Value value = getValue(name);
     if (value.hasMeaning() && value.meaning() == 18 && value instanceof StringValue) {
       return new Blob(ByteString.copyFromUtf8(getString(name)), false);
     }
-    return ((BlobValue) value).get();
+    return ((Value) value).get();
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index c68e3246f62f..17084249c646 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -9,7 +9,7 @@
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 
-import java.util.EnumMap;
+import java.util.Map;
 
 
 /**
@@ -35,7 +35,7 @@ public abstract class Query extends Serializable {
   public abstract static class Type implements java.io.Serializable {
 
     private static final long serialVersionUID = 2104157695425806623L;
-    private static final EnumMap>
+    private static final Map>
         PB_TO_INSTANCE = Maps.newEnumMap(DatastoreV1.EntityResult.ResultType.class);
 
     static final Type UNKNOWN = new Type(null, Object.class) {
@@ -127,7 +127,7 @@ boolean isAssignableFrom(Type otherType) {
       return resultClass.isAssignableFrom(otherType.resultClass);
     }
 
-    protected abstract V convert(DatastoreV1.Entity value);
+    protected abstract V convert(DatastoreV1.Entity entityPb);
 
     static Type fromPb(DatastoreV1.EntityResult.ResultType typePb) {
       return MoreObjects.firstNonNull(PB_TO_INSTANCE.get(typePb), UNKNOWN);
@@ -150,6 +150,7 @@ public String namespace() {
   @Override
   public String toString() {
     ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
+    toStringHelper.add("type", type);
     toStringHelper.add("namespace", namespace);
     toStringHelper.add("queryPb", super.toString());
     return toStringHelper.toString();
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index 5c32f028b6b3..f1f6ee2f4838 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -7,6 +7,7 @@
 import com.google.gcloud.datastore.Query.Type;
 
 import java.util.Iterator;
+import java.util.Objects;
 
 class QueryResultImpl extends AbstractIterator implements QueryResult {
 
@@ -51,7 +52,7 @@ private void sendRequest() {
     entityResultPbIter = queryResultBatchPb.getEntityResultList().iterator();
     // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
     actualType = Type.fromPb(queryResultBatchPb.getEntityResultType());
-    if (queryType == Type.PROJECTION) {
+    if (Objects.equals(queryType, Type.PROJECTION)) {
       // projection entity can represent all type of results
       actualType = Type.PROJECTION;
     }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 7cd197e27ee4..c02e7e5fc91a 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -17,7 +17,11 @@
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.io.Serializable;
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
 
 /**
  * An implementation of a Google Cloud Datastore Query that can be constructed by providing
@@ -137,7 +141,7 @@ public boolean equals(Object obj) {
         return false;
       }
       CompositeFilter other = (CompositeFilter) obj;
-      return operator.equals(other.operator)
+      return operator == other.operator
           && filters.equals(other.filters);
     }
 
@@ -228,7 +232,7 @@ public boolean equals(Object obj) {
       }
       PropertyFilter other = (PropertyFilter) obj;
       return property.equals(other.property)
-          && operator.equals(other.operator)
+          && operator == other.operator
           && Objects.equals(value, other.value);
     }
 
@@ -453,7 +457,7 @@ public boolean equals(Object obj) {
       }
       OrderBy other = (OrderBy) obj;
       return property.equals(other.property)
-          && direction.equals(other.direction);
+          && direction == other.direction;
     }
 
     public String property() {
@@ -591,7 +595,7 @@ static class BaseBuilder> {
     }
 
     @SuppressWarnings("unchecked")
-    protected B self() {
+    B self() {
       return (B) this;
     }
 
@@ -649,43 +653,41 @@ public B addOrderBy(OrderBy orderBy, OrderBy... others) {
       return self();
     }
 
-    protected B clearProjection() {
+    B clearProjection() {
       projection.clear();
       return self();
     }
 
-    protected B projection(Projection projection, Projection... others) {
+    B projection(Projection projection, Projection... others) {
       clearProjection();
       addProjection(projection, others);
       return self();
     }
 
-    protected B addProjection(Projection projection, Projection... others) {
+    B addProjection(Projection projection, Projection... others) {
       this.projection.add(projection);
-      for (Projection other : others) {
-        this.projection.add(other);
-      }
+      Collections.addAll(this.projection, others);
       return self();
     }
 
-    protected B clearGroupBy() {
+    B clearGroupBy() {
       groupBy.clear();
       return self();
     }
 
-    protected B groupBy(String property, String... others) {
+    B groupBy(String property, String... others) {
       clearGroupBy();
       addGroupBy(property, others);
       return self();
     }
 
-    protected B addGroupBy(String property, String... others) {
+    B addGroupBy(String property, String... others) {
       this.groupBy.add(property);
       Collections.addAll(this.groupBy, others);
       return self();
     }
 
-    protected B mergeFrom(DatastoreV1.Query queryPb) {
+    B mergeFrom(DatastoreV1.Query queryPb) {
       if (queryPb.getKindCount() > 0) {
         kind(queryPb.getKind(0).getName());
       }
@@ -869,7 +871,7 @@ protected boolean keyOnly() {
     return projection.size() == 1 && projection.get(0).property.equals(KEY_PROPERTY_NAME);
   }
 
-  protected List projection() {
+  List projection() {
     return projection;
   }
 
@@ -877,7 +879,7 @@ public Filter filter() {
     return filter;
   }
 
-  protected List groupBy() {
+  List groupBy() {
     return groupBy;
   }
 
@@ -961,7 +963,7 @@ protected Object fromPb(Type type, String namespace, byte[] bytesPb)
     return fromPb(type, namespace, DatastoreV1.Query.parseFrom(bytesPb));
   }
 
-  static StructuredQuery fromPb(Type type, String namespace, DatastoreV1.Query queryPb) {
+  private static StructuredQuery fromPb(Type type, String namespace, DatastoreV1.Query queryPb) {
     BaseBuilder builder;
     if (type.equals(Type.FULL)) {
       builder = builder();
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 0bd51b0e8615..44b231b81383 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -48,7 +48,7 @@ public Entity get(Key key) {
 
   @Override
   public Iterator get(Key key, Key... others) {
-    checkActive();
+    validateActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
     return datastore.get(readOptionsPb.build(), key, others);
@@ -56,7 +56,7 @@ public Iterator get(Key key, Key... others) {
 
   @Override
   public  QueryResult run(Query query) {
-    checkActive();
+    validateActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
     return datastore.run(readOptionsPb.build(), query);
@@ -69,7 +69,7 @@ public void commit() {
 
   @Override
   public void rollback() {
-    super.checkActive();
+    super.validateActive();
     if (!wasRolledback) {
       datastore.rollbackTransaction(transaction);
     }
@@ -87,8 +87,8 @@ protected String getName() {
   }
 
   @Override
-  protected void checkActive() {
-    super.checkActive();
+  protected void validateActive() {
+    super.validateActive();
     if (wasRolledback) {
       throw throwInvalidRequest(getName() + " is not active (was rolledback)");
     }
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/src/main/java/com/google/gcloud/datastore/Validator.java
index 4d4afbc120b4..d2586573d9e8 100644
--- a/src/main/java/com/google/gcloud/datastore/Validator.java
+++ b/src/main/java/com/google/gcloud/datastore/Validator.java
@@ -23,8 +23,8 @@ private Validator() {
 
   static String validateDataset(String dataset) {
     checkArgument(!Strings.isNullOrEmpty(dataset), "dataset can't be empty or null");
-    checkArgument(Validator.DATASET_PATTERN.matcher(dataset).matches(),
-          "dataset must match the following pattern: " + Validator.DATASET_PATTERN.pattern());
+    checkArgument(DATASET_PATTERN.matcher(dataset).matches(),
+        "dataset must match the following pattern: " + DATASET_PATTERN.pattern());
     return dataset;
   }
 
@@ -33,8 +33,8 @@ static String validateNamespace(String namespace) {
       checkArgument(!namespace.isEmpty(), "namespace must not be an empty string");
       checkArgument(namespace.length() <= 100,
           "namespace must not contain more than 100 characters");
-      checkArgument(Validator.NAMESPACE_PATTERN.matcher(namespace).matches(),
-          "namespace must the following pattern: " + Validator.NAMESPACE_PATTERN.pattern());
+      checkArgument(NAMESPACE_PATTERN.matcher(namespace).matches(),
+          "namespace must the following pattern: " + NAMESPACE_PATTERN.pattern());
     }
     return namespace;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index 4686624adf54..c45e8bdaa0fd 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -183,7 +183,7 @@ abstract static class BaseBuilder, B extends BaseBuilder factoryClass =
-          Class.forName("com.google.appengine.api.appidentity.AppIdentityServiceFactory");
-      Method method = factoryClass.getMethod("getAppIdentityService");
-      Object appIdentityService = method.invoke(null);
-      method = appIdentityService.getClass().getMethod("getServiceAccountName");
-      String serviceAccountName = (String) method.invoke(appIdentityService);
-      int indexOfAtSign = serviceAccountName.indexOf('@');
-      return serviceAccountName.substring(0, indexOfAtSign);
-    } catch (Exception ex) {
-      return null;
-    }
-  }
-
   public static class Builder extends ServiceOptions.Builder {
 
     private String project;
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
index 35f9613c42ae..57706ece50d1 100644
--- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -35,28 +35,28 @@ class B extends A {
     class C extends A {
       @Override
       public Object call() throws FileNotFoundException {
-        return null;
+        return "c";
       }
     }
 
     class D extends C {
       @Override
       public Object call() throws IllegalArgumentException {
-        return null;
+        return "d";
       }
     }
 
     class E extends A {
       @Override
       public String call() throws NullPointerException {
-        return null;
+        return "e";
       }
     }
 
     class F extends A {
       @Override
       public Object call() throws Error {
-        return null;
+        return "f";
       }
     }
 
@@ -93,7 +93,6 @@ private static  void assertInvalidCallable(Callable callable, ExceptionHan
     }
   }
 
-  @SuppressWarnings("serial")
   @Test
   public void testShouldTry() {
     ExceptionHandler handler = ExceptionHandler.builder().retryOn(IOException.class).build();
@@ -115,13 +114,16 @@ public void testShouldTry() {
 
     final AtomicReference before = new AtomicReference<>(RetryResult.ABORT);
     Interceptor interceptor = new Interceptor() {
+
+      private static final long serialVersionUID = 1;
+
       @Override
-      public RetryResult shouldRetry(Exception exception, RetryResult retryResult) {
+      public RetryResult afterEval(Exception exception, RetryResult retryResult) {
         return retryResult == RetryResult.ABORT ? RetryResult.RETRY : RetryResult.ABORT;
       }
 
       @Override
-      public RetryResult shouldRetry(Exception exception) {
+      public RetryResult beforeEval(Exception exception) {
         return before.get();
       }
     };
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/src/test/java/com/google/gcloud/RetryHelperTest.java
index 1fafae35fd50..d22ba9bcc43b 100644
--- a/src/test/java/com/google/gcloud/RetryHelperTest.java
+++ b/src/test/java/com/google/gcloud/RetryHelperTest.java
@@ -68,31 +68,35 @@ public void testTriesWithExceptionHandling() {
     }
     assertNull(RetryHelper.getContext());
 
-    @SuppressWarnings("serial")
-    class E1 extends Exception {}
+    class E1Exception extends Exception {
+      private static final long serialVersionUID = 3874933713392137001L;
+    }
 
-    @SuppressWarnings("serial")
-    class E2 extends E1 {}
+    class E2Exception extends E1Exception {
+      private static final long serialVersionUID = -8710227162480133598L;
+    }
 
-    @SuppressWarnings("serial")
-    class E3 extends E1 {}
+    class E3Exception extends E1Exception {
+      private static final long serialVersionUID = -7794256022024001666L;
+    }
 
-    @SuppressWarnings("serial")
-    class E4 extends E2 {}
+    class E4Exception extends E2Exception {
+      private static final long serialVersionUID = -5508018234693709156L;
+    }
 
     params = RetryParams.builder().initialRetryDelayMillis(0).retryMaxAttempts(5).build();
-    handler = ExceptionHandler.builder().retryOn(E1.class, E4.class).abortOn(E3.class).build();
-    final Iterator exceptions =
-        Arrays.asList(new E1(), new E2(), new E4(), new E3()).iterator();
+    handler = ExceptionHandler.builder().retryOn(E1Exception.class, E4Exception.class).abortOn(E3Exception.class).build();
+    final Iterator exceptions =
+        Arrays.asList(new E1Exception(), new E2Exception(), new E4Exception(), new E3Exception()).iterator();
     try {
       RetryHelper.runWithRetries(new Callable() {
-        @Override public Void call() throws E1 {
+        @Override public Void call() throws E1Exception {
           throw exceptions.next();
         }
       }, params, handler);
       fail("Exception should have been thrown");
     } catch (NonRetriableException ex) {
-      assertTrue(ex.getCause() instanceof E3);
+      assertTrue(ex.getCause() instanceof E3Exception);
     }
     assertNull(RetryHelper.getContext());
   }
@@ -108,7 +112,7 @@ public void testTriesAtLeastMinTimes() {
     final int timesToFail = 7;
     assertNull(RetryHelper.getContext());
     int attempted = RetryHelper.runWithRetries(new Callable() {
-      int timesCalled = 0;
+      int timesCalled;
       @Override public Integer call() throws IOException {
         timesCalled++;
         assertEquals(timesCalled, RetryHelper.getContext().getAttemptNumber());
@@ -151,7 +155,7 @@ public void testTriesNoMoreThanMaxTimes() {
     }
   }
 
-  private class FakeTicker extends Ticker {
+  private static class FakeTicker extends Ticker {
     private final AtomicLong nanos = new AtomicLong();
 
     // Advances the ticker value by {@code time} in {@code timeUnit}.
@@ -196,7 +200,8 @@ public void testTriesNoMoreLongerThanTotalRetryPeriod() {
         }
       }), params, handler, stopwatch);
       fail();
-    } catch (RetriesExhaustedException e) {
+    } catch (RetriesExhaustedException expected) {
+      // verify timesCalled
       assertEquals(sleepOnAttempt, timesCalled.get());
     }
   }
@@ -213,29 +218,29 @@ public void testBackoffIsExponential() {
         .retryMaxAttempts(100)
         .build();
     long sleepDuration = RetryHelper.getSleepDuration(params, 1);
-    assertTrue("" + sleepDuration, sleepDuration < 13 && sleepDuration >= 7);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 13 && sleepDuration >= 7);
     sleepDuration = RetryHelper.getSleepDuration(params, 2);
-    assertTrue("" + sleepDuration, sleepDuration < 25 && sleepDuration >= 15);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 25 && sleepDuration >= 15);
     sleepDuration = RetryHelper.getSleepDuration(params, 3);
-    assertTrue("" + sleepDuration, sleepDuration < 50 && sleepDuration >= 30);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 50 && sleepDuration >= 30);
     sleepDuration = RetryHelper.getSleepDuration(params, 4);
-    assertTrue("" + sleepDuration, sleepDuration < 100 && sleepDuration >= 60);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 100 && sleepDuration >= 60);
     sleepDuration = RetryHelper.getSleepDuration(params, 5);
-    assertTrue("" + sleepDuration, sleepDuration < 200 && sleepDuration >= 120);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 200 && sleepDuration >= 120);
     sleepDuration = RetryHelper.getSleepDuration(params, 6);
-    assertTrue("" + sleepDuration, sleepDuration < 400 && sleepDuration >= 240);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 400 && sleepDuration >= 240);
     sleepDuration = RetryHelper.getSleepDuration(params, 7);
-    assertTrue("" + sleepDuration, sleepDuration < 800 && sleepDuration >= 480);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 800 && sleepDuration >= 480);
     sleepDuration = RetryHelper.getSleepDuration(params, 8);
-    assertTrue("" + sleepDuration, sleepDuration < 1600 && sleepDuration >= 960);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 1600 && sleepDuration >= 960);
     sleepDuration = RetryHelper.getSleepDuration(params, 9);
-    assertTrue("" + sleepDuration, sleepDuration < 3200 && sleepDuration >= 1920);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 3200 && sleepDuration >= 1920);
     sleepDuration = RetryHelper.getSleepDuration(params, 10);
-    assertTrue("" + sleepDuration, sleepDuration < 6400 && sleepDuration >= 3840);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 6400 && sleepDuration >= 3840);
     sleepDuration = RetryHelper.getSleepDuration(params, 11);
-    assertTrue("" + sleepDuration, sleepDuration < 12800 && sleepDuration >= 7680);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 12800 && sleepDuration >= 7680);
     sleepDuration = RetryHelper.getSleepDuration(params, 12);
-    assertTrue("" + sleepDuration, sleepDuration < 25600 && sleepDuration >= 15360);
+    assertTrue(String.valueOf(sleepDuration), sleepDuration < 25600 && sleepDuration >= 15360);
   }
 
   @Test
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index 1d943abf172d..0dfad2ebf791 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -65,7 +65,7 @@ public class DatastoreServiceIntegrationTest {
       .set("list", LIST_VALUE2)
       .build();
   private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
-      .set("name", "koko").setNull("null").set("age", 20).build();
+      .set("name", "Dan").setNull("null").set("age", 20).build();
   private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
       .set("null", NULL_VALUE).set("partial1", PARTIAL_ENTITY2).set("partial2", ENTITY2).build();
 
@@ -372,7 +372,7 @@ public void testRunGqlQueryWithCasting() {
 
     Query query2 = GqlQuery.builder("select * from " + KIND1).build();
     QueryResult results2 = datastore.run(query2);
-    assertEquals(Entity.class, results2.resultClass());
+    assertSame(Entity.class, results2.resultClass());
     @SuppressWarnings("unchecked")
     QueryResult results3 = (QueryResult) results2;
     assertTrue(results3.hasNext());
@@ -437,7 +437,7 @@ public void testRunStructuredQuery() throws DatastoreException {
     ProjectionEntity entity = results4.next();
     assertEquals(ENTITY2.key(), entity.key());
     assertEquals(20, entity.getLong("age"));
-    assertEquals("koko", entity.getString("name"));
+    assertEquals("Dan", entity.getString("name"));
     assertEquals(2, entity.properties().size());
     assertFalse(results4.hasNext());
     EasyMock.verify(mockDatastore);
@@ -522,12 +522,12 @@ public void testGetArray() {
     Entity entity3 = result.next();
     assertEquals(ENTITY3, entity3);
     assertTrue(entity3.isNull("null"));
-    assertEquals(false, entity3.getBoolean("bool"));
+    assertFalse(entity3.getBoolean("bool"));
     assertEquals(LIST_VALUE2.get(), entity3.getList("list"));
     PartialEntity partial1 = entity3.getEntity("partial1");
     Entity partial2 = entity3.getEntity("partial2");
-    assertEquals(partial1, PARTIAL_ENTITY2);
-    assertEquals(partial2, ENTITY2);
+    assertEquals(PARTIAL_ENTITY2, partial1);
+    assertEquals(ENTITY2, partial2);
     assertEquals(Value.Type.BOOLEAN, entity3.getValue("bool").type());
     assertEquals(6, entity3.names().size());
     assertFalse(entity3.contains("bla"));
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 519b7ae3ee72..4738158eb319 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -1,5 +1,7 @@
 package com.google.gcloud.datastore;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.google.api.client.util.Strings;
 
 import java.io.BufferedOutputStream;
@@ -35,7 +37,7 @@ public class LocalGcdHelper {
   public static final String DEFAULT_DATASET = "dataset1";
   public static final int PORT = 8080;
   private static final String GCD = "gcd-v1beta2-rev1-2.1.1";
-  private static final String GCD_LOC = "/" + GCD + ".zip";
+  private static final String GCD_LOC = '/' + GCD + ".zip";
 
   private static class ProcessStreamReader extends Thread {
 
@@ -103,7 +105,7 @@ public void start() throws IOException, InterruptedException {
       }
     }
 
-    File datasetFolder = new File(gcdFolder, GCD + "/" + dataset);
+    File datasetFolder = new File(gcdFolder, GCD + '/' + dataset);
     datasetFolder.delete();
 
     // TODO: if System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
@@ -193,7 +195,7 @@ public static void main(String... args) throws IOException, InterruptedException
       switch (args[0]) {
         case "START":
           if (!isActive(DEFAULT_DATASET)) {
-            LocalGcdHelper helper = LocalGcdHelper.start(DEFAULT_DATASET);
+            LocalGcdHelper helper = start(DEFAULT_DATASET);
             try (FileWriter writer = new FileWriter(".local_gcd_helper")) {
               writer.write(helper.gcdPath.toAbsolutePath().toString());
             }
@@ -207,8 +209,8 @@ public static void main(String... args) throws IOException, InterruptedException
               String path = reader.readLine();
               deleteRecurse(Paths.get(path));
             }
-          }
           file.delete();
+          }
           return;
         default:
           break;
@@ -217,15 +219,16 @@ public static void main(String... args) throws IOException, InterruptedException
     throw new RuntimeException("expecting only START | STOP");
   }
 
-  @SuppressWarnings("BooleanMethodIsAlwaysInverted")
   public static boolean isActive(String dataset) {
     try {
       String path = "/datastore/v1beta2/datasets/" + dataset + "/lookup";
       URL url = new URL("http://localhost:" + PORT + path);
-      try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
+      try (BufferedReader reader =
+               new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) {
         return "Valid RPC".equals(reader.readLine());
       }
-    } catch (IOException ex) {
+    } catch (IOException ignore) {
+      // assume not active
       return false;
     }
   }
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index d87b64d7f480..706485b4759e 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -118,7 +118,7 @@ public class SerializationTest {
   public void testValues() throws Exception {
     for (Type type : Type.values()) {
       for (Value value : TYPE_TO_VALUES.get(type)) {
-        Value copy = serialiazeAndDeserialize(value);
+        Value copy = serializeAndDeserialize(value);
         assertEquals(value, value);
         assertEquals(value, copy);
         assertNotSame(value, copy);
@@ -130,11 +130,11 @@ public void testValues() throws Exception {
 
   @Test
   public void testTypes() throws Exception {
-    Object[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
+    Serializable[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
         ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3, PROJECTION_ENTITY,
         DATE_TIME1, BLOB1, CURSOR1, GQL1, GQL2, QUERY1, QUERY2, QUERY3};
-    for (Object obj : types) {
-      Object copy = serialiazeAndDeserialize(obj);
+    for (Serializable obj : types) {
+      Object copy = serializeAndDeserialize(obj);
       assertEquals(obj, obj);
       assertEquals(obj, copy);
       assertNotSame(obj, copy);
@@ -143,7 +143,7 @@ public void testTypes() throws Exception {
   }
 
   @SuppressWarnings("unchecked")
-  private  T serialiazeAndDeserialize(T obj) throws IOException, ClassNotFoundException {
+  private  T serializeAndDeserialize(T obj) throws IOException, ClassNotFoundException {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     try (ObjectOutputStream output = new ObjectOutputStream(bytes)) {
       output.writeObject(obj);

From 5325076aabe44284f79ab12bb107f50d40cc74f4 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 8 Jan 2015 18:34:58 -0800
Subject: [PATCH 085/732] revert wrong formatting

---
 .../java/com/google/gcloud/RetryHelper.java   |  8 ++---
 .../com/google/gcloud/datastore/BaseKey.java  |  2 +-
 .../datastore/DatastoreServiceException.java  | 32 +++++++++----------
 .../datastore/DatastoreServiceImpl.java       |  4 +--
 .../com/google/gcloud/RetryParamsTest.java    | 14 ++++----
 5 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index 3c6e23cf085c..850048a7372d 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -1,10 +1,10 @@
 package com.google.gcloud;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static java.lang.Math.max;
-import static java.lang.Math.min;
-import static java.lang.Math.pow;
-import static java.lang.Math.random;
+import static java.lang.StrictMath.max;
+import static java.lang.StrictMath.min;
+import static java.lang.StrictMath.pow;
+import static java.lang.StrictMath.random;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import com.google.common.annotations.VisibleForTesting;
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index 534f390ca67c..b96f817cbe29 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -148,7 +148,7 @@ public boolean equals(Object obj) {
     if (!(obj instanceof BaseKey)) {
       return false;
     }
-    PartialKey other = (PartialKey) obj;
+    BaseKey other = (BaseKey) obj;
     return Objects.equals(dataset(), other.dataset())
         && Objects.equals(namespace(), other.namespace())
         && Objects.equals(path(), other.path());
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index 863a8cddb6e3..d5f36ebd83ab 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -39,13 +39,13 @@ public enum Code {
     INTERNAL(false, "Server returned an error", 500),
     UNKNOWN(false, "Unknown failure", -1);
 
-    private final boolean isTransient;
-    private final String defaultMessage;
+    private final boolean retriable;
+    private final String message;
     private final int httpCode;
 
-    Code(boolean isTransient, String msg, int httpCode) {
-      this.isTransient = isTransient;
-      defaultMessage = msg;
+    Code(boolean retriable, String message, int httpCode) {
+      this.retriable = retriable;
+      this.message = message;
       this.httpCode = httpCode;
     }
 
@@ -57,12 +57,12 @@ public Integer httpCode() {
      * Returns {@code true} if this exception is transient and the same request could be retried.
      * For any retry it is highly recommended to apply an exponential backoff.
      */
-    public boolean isTransient() {
-      return isTransient;
+    public boolean isRetriable() {
+      return retriable;
     }
 
-    DatastoreServiceException translate(DatastoreException exception, String msg) {
-      return new DatastoreServiceException(this, msg, exception);
+    DatastoreServiceException translate(DatastoreException exception, String message) {
+      return new DatastoreServiceException(this, message, exception);
     }
   }
 
@@ -77,13 +77,13 @@ DatastoreServiceException translate(DatastoreException exception, String msg) {
     HTTP_TO_CODE = ImmutableMap.copyOf(httpCodes);
   }
 
-  public DatastoreServiceException(Code code, String msg, Exception cause) {
-    super(MoreObjects.firstNonNull(msg, code.defaultMessage), cause);
+  public DatastoreServiceException(Code code, String message, Exception cause) {
+    super(MoreObjects.firstNonNull(message, code.message), cause);
     this.code = code;
   }
 
-  public DatastoreServiceException(Code code, String msg) {
-    this(code, msg, null);
+  public DatastoreServiceException(Code code, String message) {
+    this(code, message, null);
   }
 
   /**
@@ -131,12 +131,12 @@ static DatastoreServiceException translateAndThrow(DatastoreException exception)
 
 
   /**
-   * Throw a DatastoreServiceException with {@code FAILED_PRECONDITION} code and the {@code msg}
+   * Throw a DatastoreServiceException with {@code FAILED_PRECONDITION} code and the {@code message}
    * in a nested exception.
    *
    * @throws DatastoreServiceException every time
    */
-  static DatastoreServiceException throwInvalidRequest(String msg, Object... params) {
-    throw new DatastoreServiceException(Code.FAILED_PRECONDITION, String.format(msg, params));
+  static DatastoreServiceException throwInvalidRequest(String massage, Object... params) {
+    throw new DatastoreServiceException(Code.FAILED_PRECONDITION, String.format(massage, params));
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 024975b8cd03..a68783c95609 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -37,8 +37,8 @@ public RetryResult afterEval(Exception exception, RetryResult retryResult) {
         @Override
         public RetryResult beforeEval(Exception exception) {
           if (exception instanceof DatastoreServiceException) {
-            boolean isTransient = ((DatastoreServiceException) exception).code().isTransient();
-            return isTransient
+            boolean isRetriable = ((DatastoreServiceException) exception).code().isRetriable();
+            return isRetriable
                 ? ExceptionHandler.Interceptor.RetryResult.RETRY
                 : ExceptionHandler.Interceptor.RetryResult.ABORT;
           }
diff --git a/src/test/java/com/google/gcloud/RetryParamsTest.java b/src/test/java/com/google/gcloud/RetryParamsTest.java
index b59324f0cb1e..744bec7b59a2 100644
--- a/src/test/java/com/google/gcloud/RetryParamsTest.java
+++ b/src/test/java/com/google/gcloud/RetryParamsTest.java
@@ -62,24 +62,24 @@ public void testSetAndCopy() {
   public void testBadSettings() {
     RetryParams.Builder builder = RetryParams.builder();
     builder.initialRetryDelayMillis(-1);
-    builder = verifyFailure(builder);
+    builder = assertFailure(builder);
     builder.maxRetryDelayMillis(RetryParams.getDefaultInstance().getInitialRetryDelayMillis() - 1);
-    builder = verifyFailure(builder);
+    builder = assertFailure(builder);
     builder.retryDelayBackoffFactor(-1);
-    builder = verifyFailure(builder);
+    builder = assertFailure(builder);
     builder.retryMinAttempts(-1);
-    builder = verifyFailure(builder);
+    builder = assertFailure(builder);
     builder.retryMaxAttempts(RetryParams.getDefaultInstance().getRetryMinAttempts() - 1);
-    builder = verifyFailure(builder);
+    builder = assertFailure(builder);
     builder.totalRetryPeriodMillis(-1);
-    builder = verifyFailure(builder);
+    builder = assertFailure(builder);
     // verify that it is OK for min and max to be equal
     builder.retryMaxAttempts(RetryParams.getDefaultInstance().getRetryMinAttempts());
     builder.maxRetryDelayMillis(RetryParams.getDefaultInstance().getInitialRetryDelayMillis());
     builder.build();
   }
 
-  private static Builder verifyFailure(Builder builder) {
+  private static Builder assertFailure(Builder builder) {
     try {
       builder.build();
       fail("Expected IllegalArgumentException");

From 0358d21fc51205df26fd6c76a8e92ee916ef45f6 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 9 Jan 2015 18:47:26 -0800
Subject: [PATCH 086/732] Provide an option for an add with auto-id allocation.

---
 .../google/gcloud/datastore/BatchWriter.java  |  14 +-
 .../gcloud/datastore/BatchWriterImpl.java     |  18 ++
 .../gcloud/datastore/DatastoreHelper.java     |  33 ++--
 .../gcloud/datastore/DatastoreReader.java     |   3 +-
 .../gcloud/datastore/DatastoreService.java    |  28 ++-
 .../datastore/DatastoreServiceException.java  |   2 +-
 .../datastore/DatastoreServiceImpl.java       | 175 ++++++++++++------
 .../gcloud/datastore/PartialEntity.java       |   4 +
 .../gcloud/datastore/StructuredQuery.java     |   5 +-
 .../google/gcloud/datastore/Transaction.java  |   2 +-
 .../gcloud/datastore/TransactionImpl.java     |  10 +-
 .../com/google/gcloud/RetryHelperTest.java    |   8 +-
 .../DatastoreServiceIntegrationTest.java      |  63 +++++--
 .../gcloud/datastore/SerializationTest.java   |   3 +-
 14 files changed, 264 insertions(+), 104 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java
index d54bc0d51086..626db5aee7de 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriter.java
@@ -28,10 +28,20 @@ public interface BatchWriter extends DatastoreWriter {
   @Override
   void add(Entity... entity);
 
+  /**
+   * Datastore add operation.
+   * This method will automatically allocate id for any entity with incomplete key.
+   *
+   * @throws IllegalArgumentException if any of the given entities is missing a key
+   * @throws DatastoreServiceException if a given entity with a complete key was already added to
+   *     this batch or if batch is no longer active
+   */
+  void add(PartialEntity... entity);
+
   /**
    * {@inheritDoc}
    * This operation will be converted to {@link #put} operation for entities that were already
-   *     added or put in this batch.
+   *     added or put in this batch
    * @throws DatastoreServiceException if an entity is marked for deletion in this batch or if
    *     batch is no longer active
    */
@@ -41,7 +51,7 @@ public interface BatchWriter extends DatastoreWriter {
   /**
    * {@inheritDoc}
    * This operation will also remove from this batch any prior writes for entities with the same
-   *     keys.
+   *     keys
    * @throws DatastoreServiceException if batch is no longer active
    */
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
index 273d46f254a3..fc41a2fb25e0 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
@@ -7,12 +7,15 @@
 
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 class BatchWriterImpl implements BatchWriter {
 
   private final Map toAdd = new LinkedHashMap<>();
+  private final List toAddAutoId = new LinkedList<>();
   private final Map toUpdate = new LinkedHashMap<>();
   private final Map toPut = new LinkedHashMap<>();
   private final Set toDelete = new LinkedHashSet<>();
@@ -59,6 +62,18 @@ public void add(Entity... entities) {
     }
   }
 
+  @Override
+  public void add(PartialEntity... entities) {
+    validateActive();
+    for (PartialEntity entity : entities) {
+      if (entity instanceof Entity) {
+        add((Entity) entity);
+      } else {
+        toAddAutoId.add(entity);
+      }
+    }
+  }
+
   @Override
   public void update(Entity... entities) {
     validateActive();
@@ -103,6 +118,9 @@ public void delete(Key... keys) {
   public void submit() {
     validateActive();
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
+    for (PartialEntity entity : toAddAutoId) {
+      mutationPb.addInsertAutoId(entity.toPb());
+    }
     for (Entity entity : toAdd.values()) {
       mutationPb.addInsert(entity.toPb());
     }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index 7fe4aeda0d96..f4b5b26f69d8 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -10,7 +10,6 @@
 /**
  * Adds some functionality to DatastoreService that should
  * be provided statically to the interface (Java 8).
- *
  */
 public class DatastoreHelper implements DatastoreService {
 
@@ -26,8 +25,8 @@ public Entity get(Key key) {
   }
 
   @Override
-  public Iterator get(Key key, Key... others) {
-    return delegate.get(key, others);
+  public Iterator get(Key... key) {
+    return delegate.get(key);
   }
 
   @Override
@@ -56,8 +55,18 @@ public Key allocateId(PartialKey key) {
   }
 
   @Override
-  public Iterator allocateId(PartialKey key, PartialKey... others) {
-    return delegate.allocateId(key, others);
+  public List allocateId(PartialKey... key) {
+    return delegate.allocateId(key);
+  }
+
+  @Override
+  public Entity add(PartialEntity entity) {
+    return delegate.add(entity);
+  }
+
+  @Override
+  public List add(PartialEntity... entity) {
+    return delegate.add(entity);
   }
 
   @Override
@@ -91,17 +100,17 @@ public KeyFactory newKeyFactory() {
    * Returns a list with a value for each given key (ordered by input).
    * A {@code null} would be returned for non-existing keys.
    */
-  public List fetch(Key key, Key... others) {
-    Iterator entities = delegate.get(key, others);
-    Map map = Maps.newHashMapWithExpectedSize(1 + others.length);
+  public List fetch(Key... keys) {
+    Iterator entities = delegate.get(keys);
+    Map map = Maps.newHashMapWithExpectedSize(keys.length);
     while (entities.hasNext()) {
       Entity entity = entities.next();
       map.put(entity.key(), entity);
     }
-    List list = new ArrayList<>(1 + others.length);
-    list.add(map.get(key));
-    for (Key other : others) {
-      list.add(map.get(other));
+    List list = new ArrayList<>(keys.length);
+    for (Key key : keys) {
+      // this will include nulls for non-existing keys
+      list.add(map.get(key));
     }
     return list;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index ea33c0104388..29a975bda442 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -22,8 +22,9 @@ public interface DatastoreReader {
    * {@link Iterator#next next} methods.
    *
    * @throws DatastoreServiceException upon failure.
+   * @see #get(Key)
    */
-  Iterator get(Key key, Key... others);
+  Iterator get(Key... key);
 
   /**
    * Submit a {@link Query} and returns its result.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index 48ad05fe5713..fb33382af74a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -1,6 +1,6 @@
 package com.google.gcloud.datastore;
 
-import java.util.Iterator;
+import java.util.List;
 
 /**
  * An interface for Google Cloud Datastore dataset.
@@ -37,10 +37,34 @@ public interface DatastoreService extends DatastoreReaderWriter {
   /**
    * Returns a list of keys using the allocated ids ordered by the input.
    *
+   * @throws DatastoreServiceException upon failure
    * @see #allocateId(PartialKey)
+   */
+  List allocateId(PartialKey... key);
+
+  /**
+   * Datastore add operation.
+   * This method will automatically allocate an id if necessary.
+   *
+   * @param entity the entity to add
+   * @return an {@code Entity} with the same properties and a key that is either newly allocated
+   *     or the same one if was already complete
+   * @throws DatastoreServiceException upon failure
+   * @throws IllegalArgumentException if the given entity is missing a key
+   */
+  Entity add(PartialEntity entity);
+
+  /**
+   * Datastore add operation.
+   * This method will automatically allocate id for any entity with incomplete key.
+   *
+   * @return a list of {@code Entity} ordered by input with the same properties and a key that is
+   *     either newly allocated or the same one if was already complete
    * @throws DatastoreServiceException upon failure
+   * @throws IllegalArgumentException if any of the given entities is missing a key
+   * @see #add(PartialKey)
    */
-  Iterator allocateId(PartialKey key, PartialKey... others);
+  List add(PartialEntity... entity);
 
   /**
    * {@inheritDoc}
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index d5f36ebd83ab..b62df718fa61 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -114,7 +114,7 @@ static DatastoreServiceException translateAndThrow(DatastoreException exception)
     String reason = "";
     if (message != null) {
       try {
-        JSONObject json = new JSONObject(new JSONTokener(exception.getMessage()));
+        JSONObject json = new JSONObject(new JSONTokener(message));
         JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
         reason = error.getString("reason");
         message = error.getString("message");
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index a68783c95609..2dd1a482d299 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -3,8 +3,13 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.api.services.datastore.client.Datastore;
 import com.google.api.services.datastore.client.DatastoreException;
+import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
 import com.google.gcloud.ExceptionHandler;
 import com.google.gcloud.RetryHelper;
 import com.google.gcloud.RetryHelper.RetryHelperException;
@@ -12,9 +17,11 @@
 import com.google.protobuf.ByteString;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
@@ -22,8 +29,6 @@
 
 final class DatastoreServiceImpl implements DatastoreService {
 
-  static final Key[] EMPTY_KEY_ARRAY = {};
-  static final PartialKey[] EMPTY_PARTIAL_KEY_ARRAY = {};
   private static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
       new ExceptionHandler.Interceptor() {
 
@@ -97,27 +102,29 @@ DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestP
 
   @Override
   public Key allocateId(PartialKey key) {
-    return allocateId(key, EMPTY_PARTIAL_KEY_ARRAY).next();
+    return allocateId(new PartialKey[]{key}).get(0);
   }
 
   @Override
-  public Iterator allocateId(PartialKey key, PartialKey... others) {
+  public List allocateId(PartialKey... keys) {
+    if (keys.length == 0) {
+      return Collections.emptyList();
+    }
     DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder();
-    requestPb.addKey(trimNameOrId(key).toPb());
-    for (PartialKey other : others) {
-      requestPb.addKey(trimNameOrId(other).toPb());
+    for (PartialKey key : keys) {
+      requestPb.addKey(trimNameOrId(key).toPb());
     }
     // TODO(ozarov): will need to populate "force" after b/18594027 is fixed.
     DatastoreV1.AllocateIdsResponse responsePb = allocateIds(requestPb.build());
-    final Iterator keys = responsePb.getKeyList().iterator();
-    return new AbstractIterator() {
-      @Override protected Key computeNext() {
-        if (keys.hasNext()) {
-          return Key.fromPb(keys.next());
-        }
-        return endOfData();
-      }
-    };
+    Iterator keyIterator = responsePb.getKeyList().iterator();
+    ImmutableList.Builder builder = ImmutableList.builder().addAll(
+        Iterators.transform(keyIterator, new Function() {
+          @Override
+          public Key apply(DatastoreV1.Key keyPb) {
+            return Key.fromPb(keyPb);
+          }
+        }));
+    return builder.build();
   }
 
   DatastoreV1.AllocateIdsResponse allocateIds(final DatastoreV1.AllocateIdsRequest requestPb) {
@@ -139,26 +146,71 @@ private PartialKey trimNameOrId(PartialKey key) {
     return key;
   }
 
+  @Override
+  public Entity add(PartialEntity entity) {
+    return add(new PartialEntity[] {entity}).get(0);
+  }
+
+  @Override
+  public List add(PartialEntity... entities) {
+    if (entities.length == 0) {
+      return Collections.emptyList();
+    }
+    DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
+    Map completeEntities = new LinkedHashMap<>();
+    for (PartialEntity entity : entities) {
+      Entity completeEntity = null;
+      if (entity instanceof  Entity) {
+        completeEntity = (Entity) entity;
+      } else if (entity.key() instanceof Key) {
+        completeEntity = entity.toEntity((Key) entity.key());
+      }
+      if (completeEntity != null) {
+        if (completeEntities.put(completeEntity.key(), completeEntity) != null) {
+          throw DatastoreServiceException.throwInvalidRequest(
+              "Duplicate entity with the key %s", entity.key());
+        }
+        mutationPb.addInsert(completeEntity.toPb());
+      } else {
+        Preconditions.checkArgument(entity.hasKey(), "entity %s is missing a key", entity);
+        mutationPb.addInsertAutoId(entity.toPb());
+      }
+    }
+    DatastoreV1.CommitResponse commitResponse = commitMutation(mutationPb);
+    Iterator allocatedKeys =
+        commitResponse.getMutationResult().getInsertAutoIdKeyList().iterator();
+    ImmutableList.Builder responseBuilder = ImmutableList.builder();
+    for (PartialEntity entity : entities) {
+      PartialKey key = entity.key();
+      Entity completeEntity = completeEntities.get(key);
+      if (completeEntity != null) {
+        responseBuilder.add(completeEntity);
+      } else {
+        responseBuilder.add(entity.toEntity(Key.fromPb(allocatedKeys.next())));
+      }
+    }
+    return responseBuilder.build();
+  }
+
   @Override
   public Entity get(Key key) {
-    Iterator iter = get(key, EMPTY_KEY_ARRAY);
-    return iter.hasNext() ? iter.next() : null;
+    return Iterators.getNext(get(new Key[]{key}), null);
   }
 
   @Override
-  public Iterator get(Key key, Key... others) {
-    return get(null, key, others);
+  public Iterator get(Key... keys) {
+    return get(null, keys);
   }
 
-  Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key key, final Key... others) {
+  Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) {
+    if (keys.length == 0) {
+      return Collections.emptyIterator();
+    }
     DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
     if (readOptionsPb != null) {
       requestPb.setReadOptions(readOptionsPb);
     }
-    LinkedHashSet dedupKeys = new LinkedHashSet<>();
-    dedupKeys.add(key);
-    dedupKeys.addAll(Arrays.asList(others));
-    for (Key k : dedupKeys) {
+    for (Key k : Sets.newLinkedHashSet(Arrays.asList(keys))) {
       requestPb.addKey(k.toPb());
     }
     return new ResultsIterator(requestPb);
@@ -212,62 +264,70 @@ DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) {
 
   @Override
     public void add(Entity... entities) {
-    DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    Set keys = new LinkedHashSet<>();
-    for (Entity entity : entities) {
-      if (!keys.add(entity.key())) {
-        throw DatastoreServiceException.throwInvalidRequest(
-            "Duplicate entity with the key %s", entity.key());
+    if (entities.length > 0) {
+      DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
+      Set keys = new LinkedHashSet<>();
+      for (Entity entity : entities) {
+        if (!keys.add(entity.key())) {
+          throw DatastoreServiceException.throwInvalidRequest(
+              "Duplicate entity with the key %s", entity.key());
+        }
+        mutationPb.addInsert(entity.toPb());
       }
-      mutationPb.addInsert(entity.toPb());
+      commitMutation(mutationPb);
     }
-    commitMutation(mutationPb);
   }
 
   @Override
   public void update(Entity... entities) {
-    DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    Map dedupEntities = new LinkedHashMap<>();
-    for (Entity entity : entities) {
-      dedupEntities.put(entity.key(), entity);
-    }
-    for (Entity entity : dedupEntities.values()) {
-      mutationPb.addUpdate(entity.toPb());
+    if (entities.length > 0) {
+      DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
+      Map dedupEntities = new LinkedHashMap<>();
+      for (Entity entity : entities) {
+        dedupEntities.put(entity.key(), entity);
+      }
+      for (Entity entity : dedupEntities.values()) {
+        mutationPb.addUpdate(entity.toPb());
+      }
+      commitMutation(mutationPb);
     }
-    commitMutation(mutationPb);
   }
 
   @Override
   public void put(Entity... entities) {
-    DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    Map dedupEntities = new LinkedHashMap<>();
-    for (Entity entity : entities) {
-      dedupEntities.put(entity.key(), entity);
-    }
-    for (Entity e : dedupEntities.values()) {
-      mutationPb.addUpsert(e.toPb());
+    if (entities.length > 0) {
+      DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
+      Map dedupEntities = new LinkedHashMap<>();
+      for (Entity entity : entities) {
+        dedupEntities.put(entity.key(), entity);
+      }
+      for (Entity e : dedupEntities.values()) {
+        mutationPb.addUpsert(e.toPb());
+      }
+      commitMutation(mutationPb);
     }
-    commitMutation(mutationPb);
   }
 
   @Override
   public void delete(Key... keys) {
-    DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    Set dedupKeys = new LinkedHashSet<>(Arrays.asList(keys));
-    for (Key key : dedupKeys) {
-      mutationPb.addDelete(key.toPb());
+    if (keys.length > 0) {
+      DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
+      Set dedupKeys = new LinkedHashSet<>(Arrays.asList(keys));
+      for (Key key : dedupKeys) {
+        mutationPb.addDelete(key.toPb());
+      }
+      commitMutation(mutationPb);
     }
-    commitMutation(mutationPb);
   }
 
-  private void commitMutation(DatastoreV1.Mutation.Builder mutationPb) {
+  private DatastoreV1.CommitResponse commitMutation(DatastoreV1.Mutation.Builder mutationPb) {
     if (options.force()) {
       mutationPb.setForce(true);
     }
     DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
     requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL);
     requestPb.setMutation(mutationPb);
-    commit(requestPb.build());
+    return commit(requestPb.build());
   }
 
   DatastoreV1.CommitResponse commit(final DatastoreV1.CommitRequest requestPb) {
@@ -290,7 +350,8 @@ DatastoreV1.BeginTransactionResponse beginTransaction(
       final DatastoreV1.BeginTransactionRequest requestPb) {
     try {
       return RetryHelper.runWithRetries(new Callable() {
-        @Override public DatastoreV1.BeginTransactionResponse call() throws DatastoreException {
+        @Override
+        public DatastoreV1.BeginTransactionResponse call() throws DatastoreException {
           return datastore.beginTransaction(requestPb);
         }
       }, retryParams, EXCEPTION_HANDLER);
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index 4f3a71955455..c1dead077d57 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -55,6 +55,10 @@ public Entity toEntity(Key key) {
     return new Entity(key, ImmutableSortedMap.copyOf(properties()));
   }
 
+  public boolean hasKey() {
+    return key != null;
+  }
+
   /**
    * Returns the key for this entity or {@code null} if it does not have one.
    */
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index c02e7e5fc91a..671e7403b511 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -868,7 +868,7 @@ public String kind() {
   }
 
   protected boolean keyOnly() {
-    return projection.size() == 1 && projection.get(0).property.equals(KEY_PROPERTY_NAME);
+    return projection.size() == 1 && KEY_PROPERTY_NAME.equals(projection.get(0).property);
   }
 
   List projection() {
@@ -963,7 +963,8 @@ protected Object fromPb(Type type, String namespace, byte[] bytesPb)
     return fromPb(type, namespace, DatastoreV1.Query.parseFrom(bytesPb));
   }
 
-  private static StructuredQuery fromPb(Type type, String namespace, DatastoreV1.Query queryPb) {
+  private static StructuredQuery fromPb(Type type, String namespace,
+      DatastoreV1.Query queryPb) {
     BaseBuilder builder;
     if (type.equals(Type.FULL)) {
       builder = builder();
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index 06c4e5a45997..21a41f896bf7 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -55,7 +55,7 @@ public interface Transaction extends DatastoreReaderWriter {
    * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
-  Iterator get(Key key, Key... others);
+  Iterator get(Key... key);
 
   /**
    * {@inheritDoc}
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 44b231b81383..d88065eca35d 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -3,6 +3,7 @@
 import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest;
 
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.collect.Iterators;
 import com.google.gcloud.datastore.TransactionOption.IsolationLevel;
 import com.google.protobuf.ByteString;
 
@@ -11,7 +12,7 @@
 import java.util.List;
 import java.util.Map;
 
-public final class TransactionImpl extends BatchWriterImpl implements Transaction {
+final class TransactionImpl extends BatchWriterImpl implements Transaction {
 
   private final ByteString transaction;
   private boolean wasRolledback;
@@ -42,16 +43,15 @@ private static BatchWriteOption[] getBatchOptions(TransactionOption... options)
 
   @Override
   public Entity get(Key key) {
-    Iterator iter = get(key, DatastoreServiceImpl.EMPTY_KEY_ARRAY);
-    return iter.hasNext() ? iter.next() : null;
+    return Iterators.getNext(get(new Key[] {key}), null);
   }
 
   @Override
-  public Iterator get(Key key, Key... others) {
+  public Iterator get(Key... keys) {
     validateActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
-    return datastore.get(readOptionsPb.build(), key, others);
+    return datastore.get(readOptionsPb.build(), keys);
   }
 
   @Override
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/src/test/java/com/google/gcloud/RetryHelperTest.java
index d22ba9bcc43b..7f89d5e654a6 100644
--- a/src/test/java/com/google/gcloud/RetryHelperTest.java
+++ b/src/test/java/com/google/gcloud/RetryHelperTest.java
@@ -85,9 +85,11 @@ class E4Exception extends E2Exception {
     }
 
     params = RetryParams.builder().initialRetryDelayMillis(0).retryMaxAttempts(5).build();
-    handler = ExceptionHandler.builder().retryOn(E1Exception.class, E4Exception.class).abortOn(E3Exception.class).build();
-    final Iterator exceptions =
-        Arrays.asList(new E1Exception(), new E2Exception(), new E4Exception(), new E3Exception()).iterator();
+    handler = ExceptionHandler.builder()
+        .retryOn(E1Exception.class, E4Exception.class)
+        .abortOn(E3Exception.class).build();
+    final Iterator exceptions = Arrays.asList(
+        new E1Exception(), new E2Exception(), new E4Exception(), new E3Exception()).iterator();
     try {
       RetryHelper.runWithRetries(new Callable() {
         @Override public Void call() throws E1Exception {
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index 0dfad2ebf791..5fc4949a3748 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -15,7 +15,6 @@
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
-
 import org.easymock.EasyMock;
 import org.junit.After;
 import org.junit.Before;
@@ -25,10 +24,8 @@
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 
 @RunWith(JUnit4.class)
 public class DatastoreServiceIntegrationTest {
@@ -250,7 +247,10 @@ public void testNewBatchWriter() {
     Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
 
     batchWriter.add(entity4, entity5);
+    batchWriter.add(PARTIAL_ENTITY1);
     batchWriter.put(ENTITY3, entity1, entity2);
+    // TODO(OZAROV): CHANGE SUBMIT TO RETURN A BATCH RESPONSE WITH GENERATED KEYS
+    // AND USE THE KEY TO VALIDATE THE WRITE
     batchWriter.submit();
     Iterator entities =
         helper.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
@@ -474,20 +474,15 @@ public void testAllocateIdArray() {
     PartialKey partialKey2 = keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
     Key key3 = keyFactory.newKey("name");
     Key key4 = keyFactory.newKey(1);
-    Iterator result =
+    List result =
         datastore.allocateId(partialKey1, partialKey2, key3, key4, partialKey1, key3);
-    Map map = new HashMap<>();
-    int count = 0;
-    while (result.hasNext()) {
-      map.put(++count, result.next());
-    }
-    assertEquals(6, map.size());
-    assertEquals(partialKey1.newKey(map.get(1).id()), map.get(1));
-    assertEquals(partialKey1.newKey(map.get(5).id()), map.get(5));
-    assertEquals(partialKey2.newKey(map.get(2).id()), map.get(2));
-    assertEquals(Key.builder(key3).id(map.get(3).id()).build(), map.get(3));
-    assertEquals(Key.builder(key3).id(map.get(6).id()).build(), map.get(6));
-    assertEquals(Key.builder(key4).id(map.get(4).id()).build(), map.get(4));
+    assertEquals(6, result.size());
+    assertEquals(partialKey1.newKey(result.get(0).id()), result.get(0));
+    assertEquals(partialKey1.newKey(result.get(4).id()), result.get(4));
+    assertEquals(partialKey2.newKey(result.get(1).id()), result.get(1));
+    assertEquals(Key.builder(key3).id(result.get(2).id()).build(), result.get(2));
+    assertEquals(Key.builder(key3).id(result.get(5).id()).build(), result.get(5));
+    assertEquals(Key.builder(key4).id(result.get(3).id()).build(), result.get(3));
   }
 
   @Test
@@ -542,7 +537,7 @@ public void testGetArray() {
   }
 
   @Test
-  public void testAdd() {
+  public void testAddEntity() {
     List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
@@ -558,6 +553,40 @@ public void testAdd() {
     assertEquals(ENTITY3, datastore.get(ENTITY3.key()));
   }
 
+  @Test
+  public void testAddPartialEntity() {
+    List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
+    assertEquals(ENTITY1, keys.get(0));
+    assertNull(keys.get(1));
+    assertEquals(2, keys.size());
+
+    try {
+      datastore.add(ENTITY1);
+      fail("Expecting a failure");
+    } catch (DatastoreServiceException expected) {
+      // expected;
+    }
+
+    PartialEntity pe = PartialEntity.builder(PARTIAL_ENTITY2).key(KEY5).build();
+    List response =
+        datastore.add(PARTIAL_ENTITY1, PARTIAL_ENTITY2, ENTITY3, PARTIAL_ENTITY1, pe);
+    assertEquals(5, response.size());
+    assertEquals(PARTIAL_ENTITY1.properties(), response.get(0).properties());
+    assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(response.get(0).key()).properties());
+    assertEquals(PARTIAL_ENTITY2.properties(), response.get(1).properties());
+    assertEquals(PARTIAL_ENTITY2.properties(), datastore.get(response.get(1).key()).properties());
+    assertSame(ENTITY3, response.get(2));
+    assertEquals(ENTITY3, datastore.get(response.get(2).key()));
+    assertEquals(PARTIAL_ENTITY1.properties(), response.get(3).properties());
+    assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(response.get(3).key()).properties());
+    assertEquals(pe.properties(), response.get(4).properties());
+    assertEquals(pe.key(), response.get(4).key());
+    assertEquals(pe.properties(), datastore.get(response.get(4).key()).properties());
+    assertEquals(pe.key(), datastore.get(response.get(4).key()).key());
+    assertEquals(pe, response.get(4));
+    assertEquals(datastore.get(response.get(4).key()), response.get(4));
+  }
+
   @Test
   public void testUpdate() {
     List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 706485b4759e..44fb398bdfb6 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -143,7 +143,8 @@ public void testTypes() throws Exception {
   }
 
   @SuppressWarnings("unchecked")
-  private  T serializeAndDeserialize(T obj) throws IOException, ClassNotFoundException {
+  private  T serializeAndDeserialize(T obj)
+      throws IOException, ClassNotFoundException {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     try (ObjectOutputStream output = new ObjectOutputStream(bytes)) {
       output.writeObject(obj);

From 48c131095dc301168daa7de478d7fb1a6eaa73be Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 9 Jan 2015 18:49:08 -0800
Subject: [PATCH 087/732] add git-demo.iml

---
 git-demo.iml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-demo.iml b/git-demo.iml
index b5c10cc72045..3afdc82b792f 100644
--- a/git-demo.iml
+++ b/git-demo.iml
@@ -10,7 +10,7 @@
       
       
     
-    
+    
     
     
     

From 44f4eb53295a3212d1cbd0c78a69fe50041c1d07 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Fri, 9 Jan 2015 23:43:20 -0800
Subject: [PATCH 088/732] removing git-demo.iml

---
 .gitignore   |  61 ++++++++++++-
 git-demo.iml | 247 ---------------------------------------------------
 2 files changed, 58 insertions(+), 250 deletions(-)
 delete mode 100644 git-demo.iml

diff --git a/.gitignore b/.gitignore
index e3d713f330e1..7a46a67d0043 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,60 @@
-/target/
-.settings
+.gitignore
+
+*.py[cod]
+*.sw[op]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+__pycache__
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+*.iml
 .idea
+.settings
 .DS_Store
-*.iml
+target
+
+# Built documentation
+docs/_build
+
+# Virtual environment
+env/
+coverage.xml
+
+# Regression test environment variables.
+regression/local_test_setup
+
+# Make sure a generated file isn't accidentally committed.
+pylintrc_reduced
+
+# Wheel directory used in Travis builds.
+gcloud-python-wheels/
diff --git a/git-demo.iml b/git-demo.iml
deleted file mode 100644
index 3afdc82b792f..000000000000
--- a/git-demo.iml
+++ /dev/null
@@ -1,247 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-      
-      
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-  
-  
-    
-    
-    
-  
-
\ No newline at end of file

From 98b239194485db4356d3d3ad09147a42002528fc Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 11 Jan 2015 21:51:07 -0800
Subject: [PATCH 089/732] batch/transaction submit/commit returns generated
 keys

---
 .../com/google/gcloud/datastore/Batch.java    | 30 +++++++
 .../{BatchWriterImpl.java => BatchImpl.java}  | 41 ++++++++--
 ...BatchWriteOption.java => BatchOption.java} | 13 ++-
 .../google/gcloud/datastore/BatchWriter.java  | 79 ------------------
 .../datastore/DatastoreBatchWriter.java       | 82 +++++++++++++++++++
 .../gcloud/datastore/DatastoreHelper.java     |  4 +-
 .../gcloud/datastore/DatastoreService.java    |  5 +-
 .../datastore/DatastoreServiceImpl.java       |  4 +-
 .../google/gcloud/datastore/Transaction.java  | 10 ++-
 .../gcloud/datastore/TransactionImpl.java     | 21 +++--
 .../gcloud/datastore/TransactionOption.java   |  8 +-
 .../DatastoreServiceIntegrationTest.java      | 31 +++----
 12 files changed, 200 insertions(+), 128 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/Batch.java
 rename src/main/java/com/google/gcloud/datastore/{BatchWriterImpl.java => BatchImpl.java} (78%)
 rename src/main/java/com/google/gcloud/datastore/{BatchWriteOption.java => BatchOption.java} (61%)
 delete mode 100644 src/main/java/com/google/gcloud/datastore/BatchWriter.java
 create mode 100644 src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java

diff --git a/src/main/java/com/google/gcloud/datastore/Batch.java b/src/main/java/com/google/gcloud/datastore/Batch.java
new file mode 100644
index 000000000000..705cec0b1139
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/Batch.java
@@ -0,0 +1,30 @@
+package com.google.gcloud.datastore;
+
+/**
+ * An interface to represent a batch of write operations.
+ * Any write operation that is applied on a batch will only be sent
+ * to the Datastore upon {@link #submit}.
+ * A usage example:
+ * 
 {@code
+ *   Entity entity1 = datastore.get(key1);
+ *   Batch batch = datastore.newBatch();
+ *   Entity entity2 = Entity.builder(key2).set("name", "John").build();
+ *   entity1 = Entity.builder(entity1).clear().setNull("bla").build();
+ *   Entity entity3 = Entity.builder(key3).set("title", "title").build();
+ *   batch.update(entity1);
+ *   batch.add(entity2, entity3);
+ *   batch.submit();
+ * } 

+ */
+public interface Batch extends DatastoreBatchWriter {
+
+  interface Response extends DatastoreBatchWriter.Response {
+  }
+
+  /**
+   * Submit the batch to the Datastore.
+   *
+   * @throws DatastoreServiceException if there was any failure or if batch is not longer active
+   */
+  Response submit();
+}
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
similarity index 78%
rename from src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
rename to src/main/java/com/google/gcloud/datastore/BatchImpl.java
index fc41a2fb25e0..57c07cc2741a 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriterImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -3,7 +3,9 @@
 import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.gcloud.datastore.BatchWriteOption.ForceWrites;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.gcloud.datastore.BatchOption.ForceWrites;
 
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -12,7 +14,7 @@
 import java.util.Map;
 import java.util.Set;
 
-class BatchWriterImpl implements BatchWriter {
+class BatchImpl implements Batch {
 
   private final Map toAdd = new LinkedHashMap<>();
   private final List toAddAutoId = new LinkedList<>();
@@ -24,10 +26,30 @@ class BatchWriterImpl implements BatchWriter {
 
   private boolean active = true;
 
-  BatchWriterImpl(DatastoreServiceImpl datastore, BatchWriteOption... options) {
+
+  class ResponseImpl implements Response {
+
+    private final DatastoreV1.CommitResponse response;
+
+    public ResponseImpl(DatastoreV1.CommitResponse response) {
+      this.response = response;
+    }
+
+    @Override
+    public List generatedKeys() {
+      return Lists.transform(response.getMutationResult().getInsertAutoIdKeyList(),
+          new Function() {
+            @Override public Key apply(DatastoreV1.Key keyPb) {
+              return Key.fromPb(keyPb);
+            }
+          });
+    }
+  }
+
+  BatchImpl(DatastoreServiceImpl datastore, BatchOption... options) {
     this.datastore = datastore;
-    Map, BatchWriteOption> optionsMap =
-        BatchWriteOption.asImmutableMap(options);
+    Map, BatchOption> optionsMap =
+        BatchOption.asImmutableMap(options);
     if (optionsMap.containsKey(ForceWrites.class)) {
       force = ((ForceWrites) optionsMap.get(ForceWrites.class)).force();
     } else {
@@ -115,7 +137,11 @@ public void delete(Key... keys) {
   }
 
   @Override
-  public void submit() {
+  public Response submit() {
+    return new ResponseImpl(commitRequest());
+  }
+
+  DatastoreV1.CommitResponse commitRequest() {
     validateActive();
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
     for (PartialEntity entity : toAddAutoId) {
@@ -138,8 +164,9 @@ public void submit() {
     }
     DatastoreV1.CommitRequest.Builder requestPb = newCommitRequest();
     requestPb.setMutation(mutationPb);
-    datastore.commit(requestPb.build());
+    DatastoreV1.CommitResponse response = datastore.commit(requestPb.build());
     active = false;
+    return response;
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java b/src/main/java/com/google/gcloud/datastore/BatchOption.java
similarity index 61%
rename from src/main/java/com/google/gcloud/datastore/BatchWriteOption.java
rename to src/main/java/com/google/gcloud/datastore/BatchOption.java
index f2bf33266252..efcda4df4645 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchWriteOption.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchOption.java
@@ -4,11 +4,11 @@
 
 import java.util.Map;
 
-public abstract class BatchWriteOption implements java.io.Serializable {
+public abstract class BatchOption implements java.io.Serializable {
 
   private static final long serialVersionUID = -3932758377282659839L;
 
-  public static final class ForceWrites extends BatchWriteOption {
+  public static final class ForceWrites extends BatchOption {
 
     private static final long serialVersionUID = 2555054296046232799L;
 
@@ -23,7 +23,7 @@ public boolean force() {
     }
   }
 
-  BatchWriteOption() {
+  BatchOption() {
     // package protected
   }
 
@@ -31,11 +31,10 @@ public static ForceWrites forceWrites() {
     return new ForceWrites(true);
   }
 
-  static Map, BatchWriteOption> asImmutableMap(
-      BatchWriteOption... options) {
-    ImmutableMap.Builder, BatchWriteOption> builder =
+  static Map, BatchOption> asImmutableMap(BatchOption... options) {
+    ImmutableMap.Builder, BatchOption> builder =
         ImmutableMap.builder();
-    for (BatchWriteOption option : options) {
+    for (BatchOption option : options) {
       builder.put(option.getClass(), option);
     }
     return builder.build();
diff --git a/src/main/java/com/google/gcloud/datastore/BatchWriter.java b/src/main/java/com/google/gcloud/datastore/BatchWriter.java
deleted file mode 100644
index 626db5aee7de..000000000000
--- a/src/main/java/com/google/gcloud/datastore/BatchWriter.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package com.google.gcloud.datastore;
-
-/**
- * An interface to represent a batch of write operations.
- * Any write operation that is applied on a batch will only be sent
- * to the Datastore upon {@link #submit} and with as few RPC calls as possible.
- * A usage example:
- * 
 {@code
- *   Entity entity1 = datastore.get(key1);
- *   BatchWriter batchWriter = datastore.newBatchWriter();
- *   Entity entity2 = Entity.builder(key2).set("name", "John").build();
- *   entity1 = Entity.builder(entity1).clear().setNull("bla").build();
- *   Entity entity3 = Entity.builder(key3).set("title", new StringValue("title")).build();
- *   batchWriter.update(entity1);
- *   batchWriter.add(entity2, entity3);
- *   batchWriter.submit();
- * } 

- */
-public interface BatchWriter extends DatastoreWriter {
-
-  /**
-   * {@inheritDoc}
-   * This operation will be converted to {@link #put} operation for entities that were already
-   *     marked for deletion in this batch.
-   * @throws DatastoreServiceException if a given entity already added to this batch or if batch
-   *     is no longer active
-   */
-  @Override
-  void add(Entity... entity);
-
-  /**
-   * Datastore add operation.
-   * This method will automatically allocate id for any entity with incomplete key.
-   *
-   * @throws IllegalArgumentException if any of the given entities is missing a key
-   * @throws DatastoreServiceException if a given entity with a complete key was already added to
-   *     this batch or if batch is no longer active
-   */
-  void add(PartialEntity... entity);
-
-  /**
-   * {@inheritDoc}
-   * This operation will be converted to {@link #put} operation for entities that were already
-   *     added or put in this batch
-   * @throws DatastoreServiceException if an entity is marked for deletion in this batch or if
-   *     batch is no longer active
-   */
-  @Override
-  void update(Entity... entity);
-
-  /**
-   * {@inheritDoc}
-   * This operation will also remove from this batch any prior writes for entities with the same
-   *     keys
-   * @throws DatastoreServiceException if batch is no longer active
-   */
-  @Override
-  void delete(Key... key);
-
-  /**
-   * {@inheritDoc}
-   * This operation will also remove from this batch any prior writes for the same entities.
-   * @throws DatastoreServiceException if batch is no longer active
-   */
-  @Override
-  void put(Entity... entity);
-
-  /**
-   * Submit the batch to the Datastore.
-   *
-   * @throws DatastoreServiceException if there was any failure or if batch is not longer active
-   */
-  void submit();
-
-  /**
-   * Returns {@code true} if batch is still active (was not submitted).
-   */
-  boolean active();
-}
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
new file mode 100644
index 000000000000..25c2409e58ea
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import java.util.List;
+
+/**
+ * An interface to represent a batch of write operations.
+ * All write operation for a batch writer will be applied to the Datastore in one RPC call.
+ */
+interface DatastoreBatchWriter extends DatastoreWriter {
+
+  interface Response {
+    List generatedKeys();
+  }
+
+  /**
+   * {@inheritDoc}
+   * This operation will be converted to {@link #put} operation for entities that were already
+   *     marked for deletion in this writer.
+   * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity already added
+   *     to this writer or if not active
+   */
+  @Override
+  void add(Entity... entity);
+
+  /**
+   * Datastore add operation.
+   * This method will automatically allocate id for any entity with an incomplete key.
+   *
+   * @throws IllegalArgumentException if any of the given entities is missing a key
+   * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity with a
+   *     complete key was already added to this writer or if not active
+   */
+  void add(PartialEntity... entity);
+
+  /**
+   * {@inheritDoc}
+   * This operation will be converted to {@link #put} operation for entities that were already
+   *     added or put in this writer
+   * @throws com.google.gcloud.datastore.DatastoreServiceException if an entity is marked for
+   *     deletion in this writer or if not active
+   */
+  @Override
+  void update(Entity... entity);
+
+  /**
+   * {@inheritDoc}
+   * This operation will also remove from this batch any prior writes for entities with the same
+   *     keys
+   * @throws com.google.gcloud.datastore.DatastoreServiceException if not active
+   */
+  @Override
+  void delete(Key... key);
+
+  /**
+   * {@inheritDoc}
+   * This operation will also remove from this writer any prior writes for the same entities.
+   * @throws com.google.gcloud.datastore.DatastoreServiceException if not active
+   */
+  @Override
+  void put(Entity... entity);
+
+  /**
+   * Returns {@code true} if still active (write operations were not sent to the Datastore).
+   */
+  boolean active();
+}
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index f4b5b26f69d8..453c3e8c6535 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -45,8 +45,8 @@ public Transaction newTransaction(TransactionOption... options) {
   }
 
   @Override
-  public BatchWriter newBatchWriter(BatchWriteOption... options) {
-    return delegate.newBatchWriter(options);
+  public Batch newBatch(BatchOption... options) {
+    return delegate.newBatch(options);
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index fb33382af74a..aba23f36a2d2 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -20,10 +20,9 @@ public interface DatastoreService extends DatastoreReaderWriter {
   Transaction newTransaction(TransactionOption... options);
 
   /**
-   * Returns a new Batch writer for processing multiple write operations
-   * in one request.
+   * Returns a new Batch for processing multiple write operations in one request.
    */
-  BatchWriter newBatchWriter(BatchWriteOption... options);
+  Batch newBatch(BatchOption... options);
 
   /**
    * Allocate a unique id for the given key.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 2dd1a482d299..ae5b5619e62f 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -70,8 +70,8 @@ public DatastoreServiceOptions options() {
   }
 
   @Override
-  public BatchWriter newBatchWriter(BatchWriteOption... options) {
-    return new BatchWriterImpl(this, options);
+  public Batch newBatch(BatchOption... options) {
+    return new BatchImpl(this, options);
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index 21a41f896bf7..fea662ead76a 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -4,7 +4,7 @@
 
 /**
  * A Google cloud datastore transaction.
- * Any write operation that is applied on a transaction will only be sent
+ * Similar to {@link Batch} any write operation that is applied on a transaction will only be sent
  * to the Datastore upon {@link #commit}. A call to {@link #rollback} will invalidate
  * the transaction and discard the changes. Any read operation that is done by a transaction
  * will be part of it and therefore a {@code commit} is guaranteed to fail if an entity
@@ -33,7 +33,10 @@
  * @see Google Cloud Datastore transactions
  *
  */
-public interface Transaction extends DatastoreReaderWriter {
+public interface Transaction extends DatastoreBatchWriter, DatastoreReaderWriter {
+
+  interface Response extends DatastoreBatchWriter.Response {
+  }
 
   /**
    * {@inheritDoc}
@@ -74,7 +77,7 @@ public interface Transaction extends DatastoreReaderWriter {
    *
    * @throws DatastoreServiceException if could not commit the transaction or if no longer active
    */
-  void commit();
+  Response commit();
 
   /**
    * Rollback the transaction.
@@ -86,5 +89,6 @@ public interface Transaction extends DatastoreReaderWriter {
   /**
    * Returns {@code true} if the transaction is still active (was not committed or rolledback).
    */
+  @Override
   boolean active();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index d88065eca35d..c2b4b3c9d5ae 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -12,11 +12,18 @@
 import java.util.List;
 import java.util.Map;
 
-final class TransactionImpl extends BatchWriterImpl implements Transaction {
+final class TransactionImpl extends BatchImpl implements Transaction {
 
   private final ByteString transaction;
   private boolean wasRolledback;
 
+  class ResponseImpl extends BatchImpl.ResponseImpl implements Transaction.Response {
+
+    public ResponseImpl(DatastoreV1.CommitResponse response) {
+      super(response);
+    }
+  }
+
   TransactionImpl(DatastoreServiceImpl datastore, TransactionOption... options) {
     super(datastore, getBatchOptions(options));
     DatastoreV1.BeginTransactionRequest.Builder requestPb =
@@ -30,15 +37,15 @@ final class TransactionImpl extends BatchWriterImpl implements Transaction {
     transaction = datastore.requestTransactionId(requestPb);
   }
 
-  private static BatchWriteOption[] getBatchOptions(TransactionOption... options) {
-    List batchOptions = new ArrayList<>(options.length);
+  private static BatchOption[] getBatchOptions(TransactionOption... options) {
+    List batchOptions = new ArrayList<>(options.length);
     for (TransactionOption option : options) {
-      BatchWriteOption batchOption = option.toBatchWriteOption();
+      BatchOption batchOption = option.toBatchWriteOption();
       if (batchOption != null) {
         batchOptions.add(batchOption);
       }
     }
-    return batchOptions.toArray(new BatchWriteOption[batchOptions.size()]);
+    return batchOptions.toArray(new BatchOption[batchOptions.size()]);
   }
 
   @Override
@@ -63,8 +70,8 @@ public  QueryResult run(Query query) {
   }
 
   @Override
-  public void commit() {
-    submit();
+  public Transaction.Response commit() {
+    return new ResponseImpl(commitRequest());
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
index 9de8ea8ff745..433787a02f69 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionOption.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
@@ -26,8 +26,8 @@ public boolean force() {
     }
 
     @Override
-    BatchWriteOption toBatchWriteOption() {
-      return new BatchWriteOption.ForceWrites(force);
+    BatchOption toBatchWriteOption() {
+      return new BatchOption.ForceWrites(force);
     }
   }
 
@@ -63,7 +63,7 @@ public Level level() {
     }
 
     @Override
-    BatchWriteOption toBatchWriteOption() {
+    BatchOption toBatchWriteOption() {
       return null;
     }
   }
@@ -94,5 +94,5 @@ static Map, TransactionOption> asImmutableMap
     return builder.build();
   }
 
-  abstract BatchWriteOption toBatchWriteOption();
+  abstract BatchOption toBatchWriteOption();
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index 5fc4949a3748..ec40a29535b8 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -239,19 +239,18 @@ private void verifyNotUsable(DatastoreWriter writer) {
   }
 
   @Test
-  public void testNewBatchWriter() {
-    BatchWriter batchWriter = datastore.newBatchWriter();
+  public void testNewBatch() {
+    Batch batch = datastore.newBatch();
     Entity entity1 = Entity.builder(ENTITY1).clear().build();
     Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
     Entity entity4 = Entity.builder(KEY4).set("value", StringValue.of("value")).build();
     Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
 
-    batchWriter.add(entity4, entity5);
-    batchWriter.add(PARTIAL_ENTITY1);
-    batchWriter.put(ENTITY3, entity1, entity2);
-    // TODO(OZAROV): CHANGE SUBMIT TO RETURN A BATCH RESPONSE WITH GENERATED KEYS
-    // AND USE THE KEY TO VALIDATE THE WRITE
-    batchWriter.submit();
+    batch.add(entity4, entity5);
+    batch.add(PARTIAL_ENTITY1);
+    batch.put(ENTITY3, entity1, entity2);
+
+    Batch.Response response = batch.submit();
     Iterator entities =
         helper.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(entity1, entities.next());
@@ -260,19 +259,23 @@ public void testNewBatchWriter() {
     assertEquals(entity4, entities.next());
     assertEquals(entity5, entities.next());
     assertFalse(entities.hasNext());
+    List generatedKeys = response.generatedKeys();
+    assertEquals(1, generatedKeys.size());
+    // todo(ozarov): uncomment after local datastore fix their bug with returned keys
+    // assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(generatedKeys.get(0)).properties());
 
     try {
-      batchWriter.submit();
+      batch.submit();
       fail("Expecting a failure");
     } catch (DatastoreServiceException ex) {
       // expected to fail
     }
-    verifyNotUsable(batchWriter);
+    verifyNotUsable(batch);
 
-    batchWriter = datastore.newBatchWriter();
-    batchWriter.delete(entity4.key(), entity5.key());
-    batchWriter.update(ENTITY1, ENTITY2, ENTITY3);
-    batchWriter.submit();
+    batch = datastore.newBatch();
+    batch.delete(entity4.key(), entity5.key());
+    batch.update(ENTITY1, ENTITY2, ENTITY3);
+    batch.submit();
     entities = helper.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(ENTITY1, entities.next());
     assertEquals(ENTITY2, entities.next());

From 86201c9c84fbc8885d6f657b9e3f86030da1f419 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 12 Jan 2015 16:14:48 -0800
Subject: [PATCH 090/732] Provide an option for an add with auto-id allocation.

---
 .../google/gcloud/datastore/BatchImpl.java    |  4 +-
 .../google/gcloud/datastore/BlobValue.java    | 42 ++++++++-------
 .../google/gcloud/datastore/BooleanValue.java | 42 ++++++++-------
 .../gcloud/datastore/DatastoreService.java    |  2 +-
 .../datastore/DatastoreServiceImpl.java       | 13 ++---
 .../gcloud/datastore/DateTimeValue.java       | 47 ++++++++--------
 .../google/gcloud/datastore/DoubleValue.java  | 42 ++++++++-------
 .../com/google/gcloud/datastore/Entity.java   |  9 ++++
 .../google/gcloud/datastore/EntityValue.java  | 47 ++++++++--------
 .../java/com/google/gcloud/datastore/Key.java | 18 +++++++
 .../com/google/gcloud/datastore/KeyValue.java | 42 ++++++++-------
 .../google/gcloud/datastore/ListValue.java    | 54 ++++++++++---------
 .../google/gcloud/datastore/LongValue.java    | 42 ++++++++-------
 .../google/gcloud/datastore/NullValue.java    | 42 ++++++++-------
 .../gcloud/datastore/PartialEntity.java       |  8 ---
 .../google/gcloud/datastore/PartialKey.java   |  9 ----
 .../com/google/gcloud/datastore/RawValue.java | 42 ++++++++-------
 .../google/gcloud/datastore/StringValue.java  | 42 ++++++++-------
 .../gcloud/datastore/TransactionImpl.java     | 12 ++---
 .../com/google/gcloud/datastore/Value.java    |  8 ++-
 .../google/gcloud/ExceptionHandlerTest.java   |  4 +-
 .../com/google/gcloud/RetryHelperTest.java    | 42 +++++++--------
 .../DatastoreServiceIntegrationTest.java      | 54 +++++++++----------
 .../gcloud/datastore/LocalGcdHelper.java      |  9 ++--
 24 files changed, 351 insertions(+), 325 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
index 57c07cc2741a..947f108aee7a 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -23,11 +23,9 @@ class BatchImpl implements Batch {
   private final Set toDelete = new LinkedHashSet<>();
   private final boolean force;
   final DatastoreServiceImpl datastore;
-
   private boolean active = true;
 
-
-  class ResponseImpl implements Response {
+  static class ResponseImpl implements Response {
 
     private final DatastoreV1.CommitResponse response;
 
diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java
index 73671e167657..a495450a4967 100644
--- a/src/main/java/com/google/gcloud/datastore/BlobValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java
@@ -11,26 +11,28 @@ public final class BlobValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return BLOB_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(Blob value) {
-      return builder(value);
-    }
-
-    @Override
-    protected Blob getValue(DatastoreV1.Value from) {
-      return new Blob(from.getBlobValue(), false);
-    }
-
-    @Override
-    protected void setValue(BlobValue from, DatastoreV1.Value.Builder to) {
-      to.setBlobValue(from.get().byteString());
-    }
-  };
+        private static final long serialVersionUID = -823515687083612387L;
+
+        @Override
+        public int getProtoFieldId() {
+          return BLOB_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(Blob value) {
+          return builder(value);
+        }
+
+        @Override
+        protected Blob getValue(DatastoreV1.Value from) {
+          return new Blob(from.getBlobValue(), false);
+        }
+
+        @Override
+        protected void setValue(BlobValue from, DatastoreV1.Value.Builder to) {
+          to.setBlobValue(from.get().byteString());
+        }
+      };
 
   public static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
index 86e080b096fb..63de0e15791c 100644
--- a/src/main/java/com/google/gcloud/datastore/BooleanValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
@@ -11,26 +11,28 @@ public final class BooleanValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return BOOLEAN_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(Boolean value) {
-      return builder(value);
-    }
-
-    @Override
-    protected Boolean getValue(DatastoreV1.Value from) {
-      return from.getBooleanValue();
-    }
-
-    @Override
-    protected void setValue(BooleanValue from, DatastoreV1.Value.Builder to) {
-      to.setBooleanValue(from.get());
-    }
-  };
+        private static final long serialVersionUID = 7080467411349092522L;
+
+        @Override
+        public int getProtoFieldId() {
+          return BOOLEAN_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(Boolean value) {
+          return builder(value);
+        }
+
+        @Override
+        protected Boolean getValue(DatastoreV1.Value from) {
+          return from.getBooleanValue();
+        }
+
+        @Override
+        protected void setValue(BooleanValue from, DatastoreV1.Value.Builder to) {
+          to.setBooleanValue(from.get());
+        }
+      };
 
   public static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index aba23f36a2d2..c4e71bcbc010 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -61,7 +61,7 @@ public interface DatastoreService extends DatastoreReaderWriter {
    *     either newly allocated or the same one if was already complete
    * @throws DatastoreServiceException upon failure
    * @throws IllegalArgumentException if any of the given entities is missing a key
-   * @see #add(PartialKey)
+   * @see #add(PartialEntity)
    */
   List add(PartialEntity... entity);
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index ae5b5619e62f..5070c21e613a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -163,7 +163,7 @@ public List add(PartialEntity... entities) {
       if (entity instanceof  Entity) {
         completeEntity = (Entity) entity;
       } else if (entity.key() instanceof Key) {
-        completeEntity = entity.toEntity((Key) entity.key());
+        completeEntity = Entity.builder((Key) entity.key(), entity).build();
       }
       if (completeEntity != null) {
         if (completeEntities.put(completeEntity.key(), completeEntity) != null) {
@@ -186,7 +186,7 @@ public List add(PartialEntity... entities) {
       if (completeEntity != null) {
         responseBuilder.add(completeEntity);
       } else {
-        responseBuilder.add(entity.toEntity(Key.fromPb(allocatedKeys.next())));
+        responseBuilder.add(Entity.builder(Key.fromPb(allocatedKeys.next()), entity).build());
       }
     }
     return responseBuilder.build();
@@ -366,11 +366,12 @@ void rollbackTransaction(ByteString transaction) {
     rollback(requestPb.build());
   }
 
-  DatastoreV1.RollbackResponse rollback(final DatastoreV1.RollbackRequest requestPb) {
+  void rollback(final DatastoreV1.RollbackRequest requestPb) {
     try {
-      return RetryHelper.runWithRetries(new Callable() {
-        @Override public DatastoreV1.RollbackResponse call() throws DatastoreException {
-          return datastore.rollback(requestPb);
+      RetryHelper.runWithRetries(new Callable() {
+        @Override public Void call() throws DatastoreException {
+          datastore.rollback(requestPb);
+          return null;
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
index fff1b35d7f29..666dd24c2dfb 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
@@ -11,29 +11,30 @@ public final class DateTimeValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(DateTime value) {
-      return builder(value);
-    }
-
-    @Override
-    protected DateTime getValue(DatastoreV1.Value from) {
-      return new DateTime(from.getTimestampMicrosecondsValue());
-    }
-
-    @Override
-    protected void setValue(DateTimeValue from, DatastoreV1.Value.Builder to) {
-      to.setTimestampMicrosecondsValue(from.get().timestampMicroseconds());
-    }
-  };
-
-  public static final class Builder
-      extends Value.BaseBuilder {
+        private static final long serialVersionUID = -5695812592049332840L;
+
+        @Override
+        public int getProtoFieldId() {
+          return TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(DateTime value) {
+          return builder(value);
+        }
+
+        @Override
+        protected DateTime getValue(DatastoreV1.Value from) {
+          return new DateTime(from.getTimestampMicrosecondsValue());
+        }
+
+        @Override
+        protected void setValue(DateTimeValue from, DatastoreV1.Value.Builder to) {
+          to.setTimestampMicrosecondsValue(from.get().timestampMicroseconds());
+        }
+      };
+
+  public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
       super(Type.DATE_TIME);
diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
index 5d5b510739f0..b5c19f3545c6 100644
--- a/src/main/java/com/google/gcloud/datastore/DoubleValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
@@ -11,26 +11,28 @@ public final class DoubleValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return DOUBLE_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(Double value) {
-      return builder(value);
-    }
-
-    @Override
-    protected Double getValue(DatastoreV1.Value from) {
-      return from.getDoubleValue();
-    }
-
-    @Override
-    protected void setValue(DoubleValue from, DatastoreV1.Value.Builder to) {
-      to.setDoubleValue(from.get());
-    }
-  };
+        private static final long serialVersionUID = 3935522813529400538L;
+
+        @Override
+        public int getProtoFieldId() {
+          return DOUBLE_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(Double value) {
+          return builder(value);
+        }
+
+        @Override
+        protected Double getValue(DatastoreV1.Value from) {
+          return from.getDoubleValue();
+        }
+
+        @Override
+        protected void setValue(DoubleValue from, DatastoreV1.Value.Builder to) {
+          to.setDoubleValue(from.get());
+        }
+      };
 
   public static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java
index 2d4ae4306942..292291792442 100644
--- a/src/main/java/com/google/gcloud/datastore/Entity.java
+++ b/src/main/java/com/google/gcloud/datastore/Entity.java
@@ -33,6 +33,11 @@ private Builder(Entity entity) {
       key = entity.key();
     }
 
+    private Builder(Key key, BaseEntity entity) {
+      super(entity);
+      this.key = key;
+    }
+
     public Builder key(Key key) {
       this.key = checkNotNull(key);
       return this;
@@ -74,4 +79,8 @@ public static Builder builder(Key key) {
   public static Builder builder(Entity copyFrom) {
     return new Builder(copyFrom);
   }
+
+  public static Builder builder(Key key, PartialEntity copyFrom) {
+    return new Builder(key, copyFrom);
+  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java
index 061cf6c57fa5..c4e46bebc8ae 100644
--- a/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -12,29 +12,30 @@ public class EntityValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return ENTITY_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(PartialEntity value) {
-      return builder(value);
-    }
-
-    @Override
-    protected PartialEntity getValue(DatastoreV1.Value from) {
-      return PartialEntity.fromPb(from.getEntityValue());
-    }
-
-    @Override
-    protected void setValue(EntityValue from, DatastoreV1.Value.Builder to) {
-      to.setEntityValue(from.get().toPb());
-    }
-  };
-
-  public static final class Builder extends
-      Value.BaseBuilder {
+        private static final long serialVersionUID = 2355075086076070931L;
+
+        @Override
+        public int getProtoFieldId() {
+          return ENTITY_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(PartialEntity value) {
+          return builder(value);
+        }
+
+        @Override
+        protected PartialEntity getValue(DatastoreV1.Value from) {
+          return PartialEntity.fromPb(from.getEntityValue());
+        }
+
+        @Override
+        protected void setValue(EntityValue from, DatastoreV1.Value.Builder to) {
+          to.setEntityValue(from.get().toPb());
+        }
+      };
+
+  public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
       super(Type.ENTITY);
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index c35efaeb6849..43ea51d83c1c 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -38,6 +38,16 @@ private Builder(String dataset, String kind, long id) {
       this.id = id;
     }
 
+    private Builder(PartialKey copyFrom, String name) {
+      super(copyFrom);
+      this.name = name;
+    }
+
+    private Builder(PartialKey copyFrom, long id) {
+      super(copyFrom);
+      this.id = id;
+    }
+
     private Builder(Key copyFrom) {
       super(copyFrom);
       if (copyFrom.hasId()) {
@@ -158,6 +168,14 @@ public static Builder builder(Key copyFrom) {
     return new Builder(copyFrom);
   }
 
+  public static Builder builder(PartialKey copyFrom, String name) {
+    return new Builder(copyFrom, name);
+  }
+
+  public static Builder builder(PartialKey copyFrom, long id) {
+    return new Builder(copyFrom, id);
+  }
+
   public static Builder builder(Key parent, String kind, String name) {
     Builder builder = builder(parent.dataset(), kind, name);
     addParentToBuilder(parent, builder);
diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java
index a97399b5006c..65163103ede4 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyValue.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java
@@ -11,26 +11,28 @@ public final class KeyValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return KEY_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(Key key) {
-      return builder(key);
-    }
-
-    @Override
-    protected Key getValue(DatastoreV1.Value from) {
-      return Key.fromPb(from.getKeyValue());
-    }
-
-    @Override
-    protected void setValue(KeyValue from, DatastoreV1.Value.Builder to) {
-      to.setKeyValue(from.get().toPb());
-    }
-  };
+        private static final long serialVersionUID = 5449133205064700403L;
+
+        @Override
+        public int getProtoFieldId() {
+          return KEY_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(Key key) {
+          return builder(key);
+        }
+
+        @Override
+        protected Key getValue(DatastoreV1.Value from) {
+          return Key.fromPb(from.getKeyValue());
+        }
+
+        @Override
+        protected void setValue(KeyValue from, DatastoreV1.Value.Builder to) {
+          to.setKeyValue(from.get().toPb());
+        }
+      };
 
   public static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java
index 5b932c8c1da1..4132858432b4 100644
--- a/src/main/java/com/google/gcloud/datastore/ListValue.java
+++ b/src/main/java/com/google/gcloud/datastore/ListValue.java
@@ -16,32 +16,34 @@ public final class ListValue extends Value>> {
   static final BaseMarshaller>, ListValue, Builder> MARSHALLER =
       new BaseMarshaller>, ListValue, Builder>() {
 
-    @Override
-    public int getProtoFieldId() {
-      return LIST_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(List> values) {
-      return builder().set(values);
-    }
-
-    @Override
-    protected List> getValue(DatastoreV1.Value from) {
-      List> properties = new ArrayList<>(from.getListValueCount());
-      for (DatastoreV1.Value valuePb : from.getListValueList()) {
-        properties.add(Value.fromPb(valuePb));
-      }
-      return properties;
-    }
-
-    @Override
-    protected void setValue(ListValue from, DatastoreV1.Value.Builder to) {
-      for (Value property : from.get()) {
-        to.addListValue(property.toPb());
-      }
-    }
-  };
+        private static final long serialVersionUID = -3193794036327640106L;
+
+        @Override
+        public int getProtoFieldId() {
+          return LIST_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(List> values) {
+          return builder().set(values);
+        }
+
+        @Override
+        protected List> getValue(DatastoreV1.Value from) {
+          List> properties = new ArrayList<>(from.getListValueCount());
+          for (DatastoreV1.Value valuePb : from.getListValueList()) {
+            properties.add(Value.fromPb(valuePb));
+          }
+          return properties;
+        }
+
+        @Override
+        protected void setValue(ListValue from, DatastoreV1.Value.Builder to) {
+          for (Value property : from.get()) {
+            to.addListValue(property.toPb());
+          }
+        }
+      };
 
   public static final class Builder extends
       Value.BaseBuilder>, ListValue, Builder> {
diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/src/main/java/com/google/gcloud/datastore/LongValue.java
index 4ab977ed96da..ac887e4c7263 100644
--- a/src/main/java/com/google/gcloud/datastore/LongValue.java
+++ b/src/main/java/com/google/gcloud/datastore/LongValue.java
@@ -11,26 +11,28 @@ public final class LongValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return INTEGER_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(Long value) {
-      return builder(value);
-    }
-
-    @Override
-    protected Long getValue(DatastoreV1.Value from) {
-      return from.getIntegerValue();
-    }
-
-    @Override
-    protected void setValue(LongValue from, DatastoreV1.Value.Builder to) {
-      to.setIntegerValue(from.get());
-    }
-  };
+        private static final long serialVersionUID = 2137414214660959845L;
+
+        @Override
+        public int getProtoFieldId() {
+          return INTEGER_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(Long value) {
+          return builder(value);
+        }
+
+        @Override
+        protected Long getValue(DatastoreV1.Value from) {
+          return from.getIntegerValue();
+        }
+
+        @Override
+        protected void setValue(LongValue from, DatastoreV1.Value.Builder to) {
+          to.setIntegerValue(from.get());
+        }
+      };
 
   public static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java
index af81c017bc72..58c91e8342cb 100644
--- a/src/main/java/com/google/gcloud/datastore/NullValue.java
+++ b/src/main/java/com/google/gcloud/datastore/NullValue.java
@@ -11,26 +11,28 @@ public final class NullValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public Builder newBuilder(Void value) {
-      return builder();
-    }
-
-    @Override
-    public int getProtoFieldId() {
-      return 0;
-    }
-
-    @Override
-    protected Void getValue(DatastoreV1.Value from) {
-      return null;
-    }
-
-    @Override
-    protected void setValue(NullValue from, DatastoreV1.Value.Builder to) {
-      // nothing to set
-    }
-  };
+        private static final long serialVersionUID = 2785573597627128832L;
+
+        @Override
+        public Builder newBuilder(Void value) {
+          return builder();
+        }
+
+        @Override
+        public int getProtoFieldId() {
+          return 0;
+        }
+
+        @Override
+        protected Void getValue(DatastoreV1.Value from) {
+          return null;
+        }
+
+        @Override
+        protected void setValue(NullValue from, DatastoreV1.Value.Builder to) {
+          // nothing to set
+        }
+      };
 
   public static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index c1dead077d57..b774e30c2a3d 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -47,14 +47,6 @@ protected PartialEntity(PartialKey key, ImmutableSortedMap> pro
     this.key = key;
   }
 
-  /**
-   * Returns a new {@link Entity} with the same properties as this one and
-   * with the given {@code key}.
-   */
-  public Entity toEntity(Key key) {
-    return new Entity(key, ImmutableSortedMap.copyOf(properties()));
-  }
-
   public boolean hasKey() {
     return key != null;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java
index fa28ed906e2f..74659534b3c1 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java
@@ -38,15 +38,6 @@ public PartialKey build() {
     super(dataset, namespace, path);
   }
 
-  public Key newKey(String name) {
-    return Key.builder(dataset(), kind(), name)
-        .namespace(namespace()).ancestors(ancestors()).build();
-  }
-
-  public Key newKey(long id) {
-    return Key.builder(dataset(), kind(), id).namespace(namespace()).ancestors(ancestors()).build();
-  }
-
   @Override
   protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(DatastoreV1.Key.parseFrom(bytesPb));
diff --git a/src/main/java/com/google/gcloud/datastore/RawValue.java b/src/main/java/com/google/gcloud/datastore/RawValue.java
index d5fc455df4a9..17e5b70b12ab 100644
--- a/src/main/java/com/google/gcloud/datastore/RawValue.java
+++ b/src/main/java/com/google/gcloud/datastore/RawValue.java
@@ -9,26 +9,28 @@ public final class RawValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public Builder newBuilder(DatastoreV1.Value value) {
-      return builder(value);
-    }
-
-    @Override
-    public int getProtoFieldId() {
-      return 0;
-    }
-
-    @Override
-    protected DatastoreV1.Value getValue(DatastoreV1.Value from) {
-      return from;
-    }
-
-    @Override
-    protected void setValue(RawValue from, DatastoreV1.Value.Builder to) {
-      to.mergeFrom(from.get());
-    }
-  };
+        private static final long serialVersionUID = 5320642719486106244L;
+
+        @Override
+        public Builder newBuilder(DatastoreV1.Value value) {
+          return builder(value);
+        }
+
+        @Override
+        public int getProtoFieldId() {
+          return 0;
+        }
+
+        @Override
+        protected DatastoreV1.Value getValue(DatastoreV1.Value from) {
+          return from;
+        }
+
+        @Override
+        protected void setValue(RawValue from, DatastoreV1.Value.Builder to) {
+          to.mergeFrom(from.get());
+        }
+      };
 
   static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java
index dad18654c8bd..ae09e39b6b6e 100644
--- a/src/main/java/com/google/gcloud/datastore/StringValue.java
+++ b/src/main/java/com/google/gcloud/datastore/StringValue.java
@@ -12,26 +12,28 @@ public final class StringValue extends Value {
   static final BaseMarshaller MARSHALLER =
       new BaseMarshaller() {
 
-    @Override
-    public int getProtoFieldId() {
-      return STRING_VALUE_FIELD_NUMBER;
-    }
-
-    @Override
-    public Builder newBuilder(String value) {
-      return builder(value);
-    }
-
-    @Override
-    protected String getValue(DatastoreV1.Value from) {
-      return from.getStringValue();
-    }
-
-    @Override
-    protected void setValue(StringValue from, DatastoreV1.Value.Builder to) {
-      to.setStringValue(from.get());
-    }
-  };
+        private static final long serialVersionUID = -359610204134164436L;
+
+        @Override
+        public int getProtoFieldId() {
+          return STRING_VALUE_FIELD_NUMBER;
+        }
+
+        @Override
+        public Builder newBuilder(String value) {
+          return builder(value);
+        }
+
+        @Override
+        protected String getValue(DatastoreV1.Value from) {
+          return from.getStringValue();
+        }
+
+        @Override
+        protected void setValue(StringValue from, DatastoreV1.Value.Builder to) {
+          to.setStringValue(from.get());
+        }
+      };
 
   public static final class Builder extends Value.BaseBuilder {
 
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index c2b4b3c9d5ae..1b33ae19b4d8 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -15,9 +15,9 @@
 final class TransactionImpl extends BatchImpl implements Transaction {
 
   private final ByteString transaction;
-  private boolean wasRolledback;
+  private boolean rolledback;
 
-  class ResponseImpl extends BatchImpl.ResponseImpl implements Transaction.Response {
+  static class ResponseImpl extends BatchImpl.ResponseImpl implements Transaction.Response {
 
     public ResponseImpl(DatastoreV1.CommitResponse response) {
       super(response);
@@ -77,15 +77,15 @@ public Transaction.Response commit() {
   @Override
   public void rollback() {
     super.validateActive();
-    if (!wasRolledback) {
+    if (!rolledback) {
       datastore.rollbackTransaction(transaction);
     }
-    wasRolledback = true;
+    rolledback = true;
   }
 
   @Override
   public boolean active() {
-    return super.active() && !wasRolledback;
+    return super.active() && !rolledback;
   }
 
   @Override
@@ -96,7 +96,7 @@ protected String getName() {
   @Override
   protected void validateActive() {
     super.validateActive();
-    if (wasRolledback) {
+    if (rolledback) {
       throw throwInvalidRequest(getName() + " is not active (was rolledback)");
     }
   }
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index c45e8bdaa0fd..ff064e87178e 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -129,11 +129,13 @@ interface Builder, B extends Builder> {
     P build();
   }
 
-  interface BuilderFactory, B extends Builder> {
+  interface BuilderFactory, B extends Builder>
+      extends java.io.Serializable {
     B newBuilder(V value);
   }
 
-  interface Marshaller, B extends Builder> {
+  interface Marshaller, B extends Builder>
+      extends java.io.Serializable {
 
     B fromProto(DatastoreV1.Value proto);
 
@@ -145,6 +147,8 @@ interface Marshaller, B extends Builder> {
   abstract static class BaseMarshaller, B extends Builder>
       implements Marshaller, BuilderFactory {
 
+    private static final long serialVersionUID = 2880767488942992985L;
+
     @Override
     public final B fromProto(DatastoreV1.Value proto) {
       B builder = newBuilder(getValue(proto));
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
index 57706ece50d1..960f293e0baa 100644
--- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -113,9 +113,7 @@ public void testShouldTry() {
     assertTrue(handler.shouldRetry(new NullPointerException()));
 
     final AtomicReference before = new AtomicReference<>(RetryResult.ABORT);
-    Interceptor interceptor = new Interceptor() {
-
-      private static final long serialVersionUID = 1;
+    @SuppressWarnings("serial") Interceptor interceptor = new Interceptor() {
 
       @Override
       public RetryResult afterEval(Exception exception, RetryResult retryResult) {
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/src/test/java/com/google/gcloud/RetryHelperTest.java
index 7f89d5e654a6..b8b0f7a4c501 100644
--- a/src/test/java/com/google/gcloud/RetryHelperTest.java
+++ b/src/test/java/com/google/gcloud/RetryHelperTest.java
@@ -42,6 +42,22 @@
  */
 public class RetryHelperTest {
 
+  static class E1Exception extends Exception {
+    private static final long serialVersionUID = 3874933713392137001L;
+  }
+
+  static class E2Exception extends E1Exception {
+    private static final long serialVersionUID = -8710227162480133598L;
+  }
+
+  static class E3Exception extends E1Exception {
+    private static final long serialVersionUID = -7794256022024001666L;
+  }
+
+  static class E4Exception extends E2Exception {
+    private static final long serialVersionUID = -5508018234693709156L;
+  }
+
   @Test
   public void testTriesWithExceptionHandling() {
     assertNull(RetryHelper.getContext());
@@ -68,22 +84,6 @@ public void testTriesWithExceptionHandling() {
     }
     assertNull(RetryHelper.getContext());
 
-    class E1Exception extends Exception {
-      private static final long serialVersionUID = 3874933713392137001L;
-    }
-
-    class E2Exception extends E1Exception {
-      private static final long serialVersionUID = -8710227162480133598L;
-    }
-
-    class E3Exception extends E1Exception {
-      private static final long serialVersionUID = -7794256022024001666L;
-    }
-
-    class E4Exception extends E2Exception {
-      private static final long serialVersionUID = -5508018234693709156L;
-    }
-
     params = RetryParams.builder().initialRetryDelayMillis(0).retryMaxAttempts(5).build();
     handler = ExceptionHandler.builder()
         .retryOn(E1Exception.class, E4Exception.class)
@@ -161,14 +161,8 @@ private static class FakeTicker extends Ticker {
     private final AtomicLong nanos = new AtomicLong();
 
     // Advances the ticker value by {@code time} in {@code timeUnit}.
-    FakeTicker advance(long time, TimeUnit timeUnit) {
-      return advance(timeUnit.toNanos(time));
-    }
-
-    // Advances the ticker value by {@code nanoseconds}.
-    FakeTicker advance(long nanoseconds) {
-      nanos.addAndGet(nanoseconds);
-      return this;
+    void advance(long time, TimeUnit timeUnit) {
+      nanos.addAndGet(timeUnit.toNanos(time));
     }
 
     @Override
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index ec40a29535b8..3d2d751e159b 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -33,16 +33,17 @@ public class DatastoreServiceIntegrationTest {
   private static final String DATASET = LocalGcdHelper.DEFAULT_DATASET;
   private static final String KIND1 = "kind1";
   private static final String KIND2 = "kind2";
+  private static final String KIND3 = "kind3";
   private static final NullValue NULL_VALUE = NullValue.of();
   private static final StringValue STR_VALUE = StringValue.of("str");
   private static final BooleanValue BOOL_VALUE = BooleanValue.builder(false).indexed(false).build();
   private static final PartialKey PARTIAL_KEY1 = PartialKey.builder(DATASET, KIND1).build();
   private static final PartialKey PARTIAL_KEY2 = PartialKey.builder(DATASET, KIND2).build();
-  private static final Key KEY1 = PARTIAL_KEY1.newKey("name");
+  private static final Key KEY1 = Key.builder(PARTIAL_KEY1, "name").build();
   private static final Key KEY2 = Key.builder(KEY1, KIND2, 1).build();
   private static final Key KEY3 = Key.builder(KEY2).name("bla").build();
-  private static final Key KEY4 = KEY2.newKey("newName1");
-  private static final Key KEY5 = KEY2.newKey("newName2");
+  private static final Key KEY4 = Key.builder(KEY2).name("newName1").build();
+  private static final Key KEY5 = Key.builder(KEY2).name("newName2").build();
   private static final KeyValue KEY_VALUE = KeyValue.of(KEY1);
   private static final ListValue LIST_VALUE1 = ListValue.builder()
       .addValue(NULL_VALUE)
@@ -54,6 +55,8 @@ public class DatastoreServiceIntegrationTest {
       .set("str", STR_VALUE).set("bool", BOOL_VALUE).set("list", LIST_VALUE1).build();
   private static final PartialEntity PARTIAL_ENTITY2 = PartialEntity.builder(PARTIAL_ENTITY1)
       .remove("str").set("bool", true).set("list", LIST_VALUE1.get()).build();
+  private static final PartialEntity PARTIAL_ENTITY3 = PartialEntity.builder(PARTIAL_ENTITY1)
+      .key(PartialKey.builder(DATASET, KIND3).build()).build();
   private static final Entity ENTITY1 = Entity.builder(KEY1)
       .set("str", STR_VALUE)
       .set("date", DATE_TIME_VALUE)
@@ -247,7 +250,7 @@ public void testNewBatch() {
     Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
 
     batch.add(entity4, entity5);
-    batch.add(PARTIAL_ENTITY1);
+    batch.add(PARTIAL_ENTITY3);
     batch.put(ENTITY3, entity1, entity2);
 
     Batch.Response response = batch.submit();
@@ -459,15 +462,15 @@ public void testAllocateId() {
     assertEquals(key1.kind(), pk1.kind());
     assertTrue(key1.hasId());
     assertFalse(key1.hasName());
-    assertEquals(pk1.newKey(key1.id()), key1);
+    assertEquals(Key.builder(pk1, key1.id()).build(), key1);
 
     Key key2 = datastore.allocateId(pk1);
     assertNotEquals(key1, key2);
-    assertEquals(pk1.newKey(key2.id()), key2);
+    assertEquals(Key.builder(pk1, key2.id()).build(), key2);
 
     Key key3 = datastore.allocateId(key1);
     assertNotEquals(key1, key3);
-    assertEquals(pk1.newKey(key3.id()), key3);
+    assertEquals(Key.builder(pk1, key3.id()).build(), key3);
   }
 
   @Test
@@ -480,9 +483,9 @@ public void testAllocateIdArray() {
     List result =
         datastore.allocateId(partialKey1, partialKey2, key3, key4, partialKey1, key3);
     assertEquals(6, result.size());
-    assertEquals(partialKey1.newKey(result.get(0).id()), result.get(0));
-    assertEquals(partialKey1.newKey(result.get(4).id()), result.get(4));
-    assertEquals(partialKey2.newKey(result.get(1).id()), result.get(1));
+    assertEquals(Key.builder(partialKey1, result.get(0).id()).build(), result.get(0));
+    assertEquals(Key.builder(partialKey1, result.get(4).id()).build(), result.get(4));
+    assertEquals(Key.builder(partialKey2, result.get(1).id()).build(), result.get(1));
     assertEquals(Key.builder(key3).id(result.get(2).id()).build(), result.get(2));
     assertEquals(Key.builder(key3).id(result.get(5).id()).build(), result.get(5));
     assertEquals(Key.builder(key4).id(result.get(3).id()).build(), result.get(3));
@@ -571,23 +574,20 @@ public void testAddPartialEntity() {
     }
 
     PartialEntity pe = PartialEntity.builder(PARTIAL_ENTITY2).key(KEY5).build();
-    List response =
-        datastore.add(PARTIAL_ENTITY1, PARTIAL_ENTITY2, ENTITY3, PARTIAL_ENTITY1, pe);
-    assertEquals(5, response.size());
-    assertEquals(PARTIAL_ENTITY1.properties(), response.get(0).properties());
-    assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(response.get(0).key()).properties());
-    assertEquals(PARTIAL_ENTITY2.properties(), response.get(1).properties());
-    assertEquals(PARTIAL_ENTITY2.properties(), datastore.get(response.get(1).key()).properties());
-    assertSame(ENTITY3, response.get(2));
-    assertEquals(ENTITY3, datastore.get(response.get(2).key()));
-    assertEquals(PARTIAL_ENTITY1.properties(), response.get(3).properties());
-    assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(response.get(3).key()).properties());
-    assertEquals(pe.properties(), response.get(4).properties());
-    assertEquals(pe.key(), response.get(4).key());
-    assertEquals(pe.properties(), datastore.get(response.get(4).key()).properties());
-    assertEquals(pe.key(), datastore.get(response.get(4).key()).key());
-    assertEquals(pe, response.get(4));
-    assertEquals(datastore.get(response.get(4).key()), response.get(4));
+    List response = datastore.add(PARTIAL_ENTITY3, ENTITY3, PARTIAL_ENTITY3, pe);
+    assertEquals(4, response.size());
+    assertEquals(PARTIAL_ENTITY3.properties(), response.get(0).properties());
+    assertEquals(PARTIAL_ENTITY3.properties(), datastore.get(response.get(0).key()).properties());
+    assertSame(ENTITY3, response.get(1));
+    assertEquals(ENTITY3, datastore.get(response.get(1).key()));
+    assertEquals(PARTIAL_ENTITY3.properties(), response.get(2).properties());
+    assertEquals(PARTIAL_ENTITY3.properties(), datastore.get(response.get(2).key()).properties());
+    assertEquals(pe.properties(), response.get(3).properties());
+    assertEquals(pe.key(), response.get(3).key());
+    assertEquals(pe.properties(), datastore.get(response.get(3).key()).properties());
+    assertEquals(pe.key(), datastore.get(response.get(3).key()).key());
+    assertEquals(pe, response.get(3));
+    assertEquals(datastore.get(response.get(3).key()), response.get(3));
   }
 
   @Test
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 4738158eb319..64fe814e5140 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -106,7 +106,7 @@ public void start() throws IOException, InterruptedException {
     }
 
     File datasetFolder = new File(gcdFolder, GCD + '/' + dataset);
-    datasetFolder.delete();
+    deleteRecurse(datasetFolder.toPath());
 
     // TODO: if System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
     Process temp = new ProcessBuilder()
@@ -209,7 +209,7 @@ public static void main(String... args) throws IOException, InterruptedException
               String path = reader.readLine();
               deleteRecurse(Paths.get(path));
             }
-          file.delete();
+            file.delete();
           }
           return;
         default:
@@ -221,8 +221,9 @@ public static void main(String... args) throws IOException, InterruptedException
 
   public static boolean isActive(String dataset) {
     try {
-      String path = "/datastore/v1beta2/datasets/" + dataset + "/lookup";
-      URL url = new URL("http://localhost:" + PORT + path);
+      StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(PORT);
+      urlBuilder.append("/datastore/v1beta2/datasets/").append(dataset).append("/lookup");
+      URL url = new URL(urlBuilder.toString());
       try (BufferedReader reader =
                new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) {
         return "Valid RPC".equals(reader.readLine());

From fdac9a74cde9627cfd508a8a0f85e16e3d4c83ff Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 14 Jan 2015 10:46:44 -0800
Subject: [PATCH 091/732] replace local gcd hack with a fixed version of gcd

---
 .../com/google/gcloud/storage/Bucket.java     |   8 +-
 .../java/com/google/gcloud/storage/Cors.java  | 122 ++++++++++++++++++
 .../gcloud/datastore/LocalGcdHelper.java      |   2 +-
 3 files changed, 129 insertions(+), 3 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/Cors.java

diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 2149cafca6f6..be2653494fd9 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -4,15 +4,19 @@
 
 public interface Bucket {
 
+  String id();
+
   String name();
 
   Acl acl();
 
-  void updateAcl(Acl acl);
-
   Acl defaultObjectAcl();
 
+  Cors cors();
+
   void updateDefaultObjectAcl();
+  void updateAcl(Acl acl);
+
 
 
 
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
new file mode 100644
index 000000000000..71e87da50fd3
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import com.google.common.collect.ImmutableList;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+
+public class Cors {
+
+  private final Integer maxAgeSeconds;
+  private final ImmutableList methods;
+  private final ImmutableList origins;
+  private final ImmutableList responseHeaders;
+
+  public enum Method {
+    ANY, GET, HEAD, PUT, POST, DELETE
+  }
+
+  public static class Origin {
+
+    private final URI uri;
+
+    public static final Origin ANY = new Origin();
+
+    private Origin() {
+      uri = null;
+    }
+
+    public Origin(String scheme, String host, int port) {
+      try {
+        this.uri = new URI(scheme, null, host, port, null, null, null);
+      } catch (URISyntaxException e) {
+        throw new IllegalArgumentException(e);
+      }
+
+    }
+
+    @Override
+    public String toString() {
+      return uri == null ? "*" : uri.toString();
+    }
+  }
+
+  public static final class Builder {
+
+    private Integer maxAgeSeconds;
+    private ImmutableList methods;
+    private ImmutableList origins;
+    private ImmutableList responseHeaders;
+
+    private Builder() {
+    }
+
+    public Builder maxAgeSeconds(Integer maxAgeSeconds) {
+      this.maxAgeSeconds = maxAgeSeconds;
+      return this;
+    }
+
+    public Builder methods(Method... methods) {
+      this.methods = ImmutableList.copyOf(methods);
+      return this;
+    }
+
+    public Builder origins(Origin... origins) {
+      this.origins = ImmutableList.copyOf(origins);
+      return this;
+    }
+
+    public Builder responseHeaders(String... responseHeaders) {
+      this.responseHeaders = ImmutableList.copyOf(responseHeaders);
+      return this;
+    }
+
+    public Cors build() {
+      return new Cors(this);
+    }
+  }
+
+  private Cors(Builder builder) {
+    this.maxAgeSeconds = builder.maxAgeSeconds;
+    this.methods = builder.methods;
+    this.origins = builder.origins;
+    this.responseHeaders = builder.responseHeaders;
+  }
+
+  public Integer maxAgeSeconds() {
+    return maxAgeSeconds;
+  }
+
+  public List methods() {
+    return methods;
+  }
+
+  public List origins() {
+    return origins;
+  }
+
+  public List responseHeaders() {
+    return responseHeaders;
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 64fe814e5140..279cb0b2392e 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -36,7 +36,7 @@ public class LocalGcdHelper {
 
   public static final String DEFAULT_DATASET = "dataset1";
   public static final int PORT = 8080;
-  private static final String GCD = "gcd-v1beta2-rev1-2.1.1";
+  private static final String GCD = "gcd-head";
   private static final String GCD_LOC = '/' + GCD + ".zip";
 
   private static class ProcessStreamReader extends Thread {

From f7d44d5d35bd120392512767e974eb0715254377 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 14 Jan 2015 10:47:23 -0800
Subject: [PATCH 092/732] replace local gcd hack with a fixed version of gcd

---
 .../DatastoreServiceIntegrationTest.java      | 49 +------------------
 1 file changed, 2 insertions(+), 47 deletions(-)

diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index 3d2d751e159b..4af76c5ca07e 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -8,14 +8,12 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import com.google.api.services.datastore.DatastoreV1;
-import com.google.api.services.datastore.client.Datastore;
 import com.google.api.services.datastore.client.DatastoreException;
 import com.google.gcloud.datastore.Query.Type;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
-import org.easymock.EasyMock;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -264,8 +262,7 @@ public void testNewBatch() {
     assertFalse(entities.hasNext());
     List generatedKeys = response.generatedKeys();
     assertEquals(1, generatedKeys.size());
-    // todo(ozarov): uncomment after local datastore fix their bug with returned keys
-    // assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(generatedKeys.get(0)).properties());
+    assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(generatedKeys.get(0)).properties());
 
     try {
       batch.submit();
@@ -335,26 +332,6 @@ public void testRunGqlQueryNoCasting() throws DatastoreException {
     GqlQuery projectionQuery = GqlQuery.builder(
         Type.PROJECTION, "select str, date from " + KIND1).build();
 
-    // this hack is needed because of b/18806697
-    DatastoreV1.RunQueryRequest.Builder requestPb = DatastoreV1.RunQueryRequest.newBuilder();
-    requestPb.setGqlQuery(projectionQuery.toPb());
-    requestPb.setPartitionId(DatastoreV1.PartitionId.newBuilder().setDatasetId(DATASET).build());
-    DatastoreV1.RunQueryResponse responsePb =
-        ((DatastoreServiceImpl) datastore).runQuery(requestPb.build());
-    DatastoreV1.RunQueryResponse.Builder responsePbBuilder = responsePb.toBuilder();
-    responsePbBuilder.getBatchBuilder()
-        .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();
-    Datastore mockDatastore = EasyMock.createMock(Datastore.class);
-    DatastoreV1.EntityResult found =
-        DatastoreV1.EntityResult.newBuilder().setEntity(ENTITY1.toPb()).build();
-    EasyMock.expect(mockDatastore.lookup(EasyMock.anyObject()))
-        .andReturn(DatastoreV1.LookupResponse.newBuilder().addFound(found).build());
-    EasyMock.expect(mockDatastore.runQuery(requestPb.build())).andReturn(responsePbBuilder.build());
-    EasyMock.replay(mockDatastore);
-    datastore = DatastoreServiceFactory.getDefault(
-        datastore.options().toBuilder().datastore(mockDatastore).build());
-    // end of hack
-
     QueryResult projectionResult = datastore.run(projectionQuery);
     assertTrue(projectionResult.hasNext());
     projectionEntity = projectionResult.next();
@@ -364,7 +341,6 @@ public void testRunGqlQueryNoCasting() throws DatastoreException {
         projectionEntity.getLong("date"));
     assertEquals(2, projectionEntity.names().size());
     assertFalse(projectionResult.hasNext());
-    EasyMock.verify(mockDatastore);
   }
 
   @Test
@@ -418,25 +394,6 @@ public void testRunStructuredQuery() throws DatastoreException {
         .orderBy(OrderBy.asc("age"))
         .limit(10)
         .build();
-    // this hack is needed because of b/18806697
-    DatastoreV1.RunQueryRequest.Builder requestPb = DatastoreV1.RunQueryRequest.newBuilder();
-    requestPb.setQuery(projectionQuery.toPb());
-    requestPb.setPartitionId(DatastoreV1.PartitionId.newBuilder().setDatasetId(DATASET).build());
-    DatastoreV1.RunQueryResponse responsePb =
-        ((DatastoreServiceImpl) datastore).runQuery(requestPb.build());
-    DatastoreV1.RunQueryResponse.Builder responsePbBuilder = responsePb.toBuilder();
-    responsePbBuilder.getBatchBuilder()
-        .setEntityResultType(DatastoreV1.EntityResult.ResultType.PROJECTION).build();
-    Datastore mockDatastore = EasyMock.createMock(Datastore.class);
-    DatastoreV1.EntityResult missing =
-        DatastoreV1.EntityResult.newBuilder().setEntity(ENTITY1.toPb()).build();
-    EasyMock.expect(mockDatastore.lookup(EasyMock.anyObject()))
-        .andReturn(DatastoreV1.LookupResponse.newBuilder().addMissing(missing).build());
-    EasyMock.expect(mockDatastore.runQuery(requestPb.build())).andReturn(responsePbBuilder.build());
-    EasyMock.replay(mockDatastore);
-    datastore = DatastoreServiceFactory.getDefault(
-        datastore.options().toBuilder().datastore(mockDatastore).build());
-    // end of hack
 
     QueryResult results4 = datastore.run(projectionQuery);
     assertTrue(results4.hasNext());
@@ -446,8 +403,6 @@ public void testRunStructuredQuery() throws DatastoreException {
     assertEquals("Dan", entity.getString("name"));
     assertEquals(2, entity.properties().size());
     assertFalse(results4.hasNext());
-    EasyMock.verify(mockDatastore);
-
     // TODO(ozarov): construct a test to verify nextQuery/pagination
   }
 

From 01e0ff7c9ff88390432fdfbb376cc9c22c5e53cb Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 20 Jan 2015 15:23:21 -0800
Subject: [PATCH 093/732] Adding copyright and README and CONTRIBUTIONS files.

---
 .gitignore                                    | 33 +---------
 README.md                                     | 65 ++++++++++++++++++-
 .../java/com/google/gcloud/AuthConfig.java    | 16 +++++
 .../com/google/gcloud/ExceptionHandler.java   | 16 +++++
 .../java/com/google/gcloud/RetryHelper.java   | 16 +++++
 .../java/com/google/gcloud/RetryParams.java   | 16 +++++
 .../com/google/gcloud/ServiceOptions.java     | 16 +++++
 .../google/gcloud/datastore/BaseEntity.java   | 16 +++++
 .../com/google/gcloud/datastore/BaseKey.java  | 16 +++++
 .../com/google/gcloud/datastore/Batch.java    | 16 +++++
 .../google/gcloud/datastore/BatchImpl.java    | 16 +++++
 .../google/gcloud/datastore/BatchOption.java  | 16 +++++
 .../com/google/gcloud/datastore/Blob.java     | 16 +++++
 .../google/gcloud/datastore/BlobValue.java    | 16 +++++
 .../google/gcloud/datastore/BooleanValue.java | 16 +++++
 .../com/google/gcloud/datastore/Cursor.java   | 16 +++++
 .../gcloud/datastore/DatastoreHelper.java     | 16 +++++
 .../gcloud/datastore/DatastoreReader.java     | 16 +++++
 .../datastore/DatastoreReaderWriter.java      | 16 +++++
 .../gcloud/datastore/DatastoreService.java    | 16 +++++
 .../datastore/DatastoreServiceException.java  | 16 +++++
 .../datastore/DatastoreServiceFactory.java    | 16 +++++
 .../datastore/DatastoreServiceImpl.java       | 16 +++++
 .../datastore/DatastoreServiceOptions.java    | 16 +++++
 .../gcloud/datastore/DatastoreWriter.java     | 16 +++++
 .../com/google/gcloud/datastore/DateTime.java | 16 +++++
 .../gcloud/datastore/DateTimeValue.java       | 16 +++++
 .../google/gcloud/datastore/DoubleValue.java  | 16 +++++
 .../com/google/gcloud/datastore/Entity.java   | 16 +++++
 .../google/gcloud/datastore/EntityValue.java  | 16 +++++
 .../com/google/gcloud/datastore/GqlQuery.java | 16 +++++
 .../java/com/google/gcloud/datastore/Key.java | 16 +++++
 .../google/gcloud/datastore/KeyFactory.java   | 16 +++++
 .../com/google/gcloud/datastore/KeyValue.java | 16 +++++
 .../google/gcloud/datastore/ListValue.java    | 16 +++++
 .../google/gcloud/datastore/LongValue.java    | 16 +++++
 .../google/gcloud/datastore/NullValue.java    | 16 +++++
 .../gcloud/datastore/PartialEntity.java       | 16 +++++
 .../google/gcloud/datastore/PartialKey.java   | 16 +++++
 .../google/gcloud/datastore/PathElement.java  | 16 +++++
 .../gcloud/datastore/ProjectionEntity.java    | 16 +++++
 .../com/google/gcloud/datastore/Query.java    | 16 +++++
 .../google/gcloud/datastore/QueryResult.java  | 16 +++++
 .../gcloud/datastore/QueryResultImpl.java     | 16 +++++
 .../com/google/gcloud/datastore/RawValue.java | 16 +++++
 .../google/gcloud/datastore/Serializable.java | 16 +++++
 .../google/gcloud/datastore/StringValue.java  | 16 +++++
 .../gcloud/datastore/StructuredQuery.java     | 16 +++++
 .../google/gcloud/datastore/Transaction.java  | 16 +++++
 .../gcloud/datastore/TransactionImpl.java     | 16 +++++
 .../gcloud/datastore/TransactionOption.java   | 16 +++++
 .../google/gcloud/datastore/Validator.java    | 16 +++++
 .../com/google/gcloud/datastore/Value.java    | 16 +++++
 .../google/gcloud/datastore/package-info.java | 16 +++++
 .../java/com/google/gcloud/storage/Acl.java   | 16 +++++
 .../com/google/gcloud/storage/Bucket.java     | 16 +++++
 .../java/com/google/gcloud/storage/Key.java   | 16 +++++
 .../google/gcloud/storage/StorageObject.java  | 16 +++++
 .../google/gcloud/storage/StorageService.java | 16 +++++
 .../gcloud/storage/StorageServiceFactory.java | 16 +++++
 .../gcloud/storage/StorageServiceImpl.java    | 16 +++++
 .../gcloud/storage/StorageServiceOptions.java | 16 +++++
 .../google/gcloud/storage/package-info.java   | 16 +++++
 .../google/gcloud/ExceptionHandlerTest.java   | 16 +++++
 .../com/google/gcloud/RetryHelperTest.java    |  4 +-
 .../com/google/gcloud/RetryParamsTest.java    | 16 +++++
 .../DatastoreServiceIntegrationTest.java      | 16 +++++
 .../gcloud/datastore/LocalGcdHelper.java      | 16 +++++
 .../gcloud/datastore/SerializationTest.java   | 16 +++++
 69 files changed, 1124 insertions(+), 34 deletions(-)

diff --git a/.gitignore b/.gitignore
index 7a46a67d0043..355e4b02954e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,29 +1,12 @@
 .gitignore
 
-*.py[cod]
-*.sw[op]
-
-# C extensions
-*.so
-
 # Packages
-*.egg
-*.egg-info
 dist
 build
-eggs
-parts
 bin
 var
 sdist
-develop-eggs
-.installed.cfg
-lib
-lib64
-__pycache__
-
-# Installer logs
-pip-log.txt
+target
 
 # Unit test / coverage reports
 .coverage
@@ -41,20 +24,10 @@ nosetests.xml
 .idea
 .settings
 .DS_Store
-target
 
 # Built documentation
-docs/_build
-
-# Virtual environment
-env/
-coverage.xml
-
-# Regression test environment variables.
-regression/local_test_setup
+docs/
 
-# Make sure a generated file isn't accidentally committed.
-pylintrc_reduced
 
 # Wheel directory used in Travis builds.
-gcloud-python-wheels/
+gcloud-java-wheels/
diff --git a/README.md b/README.md
index 157f813e551a..3b9e93c1492b 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,63 @@
-git-demo
-========
+Google Cloud for Java
+=====================
+
+[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
+
+Java idiomatic client for Google Cloud Platform services. Supported APIs include:
+
+ * Google Cloud Datastore
+
+
+> Note: This package is a work-in-progress, and may occasionally
+> make backwards-incompatible changes.
+
+Documentation and examples are available [here](https://github.com/GoogleCloudePlatform/gcloud-java/gh-pages/docs).
+
+## Google Cloud Datastore
+
+[Google Cloud Datastore][cloud-datastore] ([docs][cloud-datastore-docs]) is a fully
+managed, schemaless database for storing non-relational data. Cloud Datastore
+automatically scales with your users and supports ACID transactions, high availability
+of reads and writes, strong consistency for reads and ancestor queries, and eventual
+consistency for all other queries.
+
+Follow the [activation instructions][cloud-datastore-activation] to use the Google
+Cloud Datastore API with your project.
+
+    import com.google.gcloud.datastore.DatastoreService;
+    import com.google.gcloud.datastore.DatastoreServiceFactory;
+    import com.google.gcloud.datastore.DatastoreServiceOptions;
+    import com.google.gcloud.datastore.Entity;
+    import com.google.gcloud.datastore.Key;
+    import com.google.gcloud.datastore.KeyFactory;
+
+    DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset("...").build();
+    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
+    KeyFactory keyFactory = new KeyFactory(datastore).kind("...");
+    Key key = keyFactory.newKey(keyName);
+    Entity entity = datastore.get(key);
+    if (entity == null) {
+      entity = Entity.builder(key)
+          .set("name", "John Do")
+          .set("age", 30)
+          .set("updated", false)
+          .build();
+      datastore.put(entity);
+    }
+
+## Contributing
+
+Contributions are welcome. Please, see the
+[CONTRIBUTING](https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md)
+document for details.
+
+[cloud-datastore]: https://cloud.google.com/datastore/
+[cloud-datastore-docs]: https://cloud.google.com/datastore/docs
+[cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate
+
+[cloud-pubsub]: https://cloud.google.com/pubsub/
+[cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs
+
+[cloud-storage]: https://cloud.google.com/storage/
+[cloud-storage-docs]: https://cloud.google.com/storage/docs/overview
+[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthConfig.java
index e1c5646c66d9..c2accdf438f8 100644
--- a/src/main/java/com/google/gcloud/AuthConfig.java
+++ b/src/main/java/com/google/gcloud/AuthConfig.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index 7330a84ba06d..0e3f73d590fd 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index 850048a7372d..7e354a4145d3 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/RetryParams.java b/src/main/java/com/google/gcloud/RetryParams.java
index 4da4c8399c8e..c7f25926eb9f 100644
--- a/src/main/java/com/google/gcloud/RetryParams.java
+++ b/src/main/java/com/google/gcloud/RetryParams.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud;
 
 import static com.google.common.base.Preconditions.checkArgument;
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index b34875f5f87f..1c51b348abc6 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud;
 
 
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index a760d98fa90e..2a132a235bf8 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.gcloud.datastore.BlobValue.of;
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index b96f817cbe29..90736614b5e1 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.gcloud.datastore.Validator.validateDataset;
diff --git a/src/main/java/com/google/gcloud/datastore/Batch.java b/src/main/java/com/google/gcloud/datastore/Batch.java
index 705cec0b1139..0a78ad3dff71 100644
--- a/src/main/java/com/google/gcloud/datastore/Batch.java
+++ b/src/main/java/com/google/gcloud/datastore/Batch.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 /**
diff --git a/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
index 947f108aee7a..4f24c2ad77df 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest;
diff --git a/src/main/java/com/google/gcloud/datastore/BatchOption.java b/src/main/java/com/google/gcloud/datastore/BatchOption.java
index efcda4df4645..362a74e96c79 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchOption.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchOption.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.common.collect.ImmutableMap;
diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java
index b8d6892d201f..42b79cc3650d 100644
--- a/src/main/java/com/google/gcloud/datastore/Blob.java
+++ b/src/main/java/com/google/gcloud/datastore/Blob.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkArgument;
diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java
index a495450a4967..4dcb80163ae4 100644
--- a/src/main/java/com/google/gcloud/datastore/BlobValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.BLOB_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
index 63de0e15791c..ac62d2d3fb27 100644
--- a/src/main/java/com/google/gcloud/datastore/BooleanValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.BOOLEAN_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java
index 25b35f6bcfe6..037e0ed7ed46 100644
--- a/src/main/java/com/google/gcloud/datastore/Cursor.java
+++ b/src/main/java/com/google/gcloud/datastore/Cursor.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index 453c3e8c6535..0b0558e565fd 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.common.collect.Maps;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index 29a975bda442..5c9ceb19cc35 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import java.util.Iterator;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
index 52304f4e7e27..c64f86a8d0a3 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index c4e71bcbc010..9770005dbc80 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import java.util.List;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index b62df718fa61..9f1b4d6ad757 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.client.DatastoreException;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
index 9a8bb1dd82bc..ca87e9cdad9b 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 5070c21e613a..c100e45e548a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 262d8f81060f..56acdb340c28 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
index 5c87a49424ba..69e911c65e14 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 /**
diff --git a/src/main/java/com/google/gcloud/datastore/DateTime.java b/src/main/java/com/google/gcloud/datastore/DateTime.java
index a2e8c6398351..3b7e85a2e736 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTime.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTime.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
index 666dd24c2dfb..f6c25b9d0353 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.TIMESTAMP_MICROSECONDS_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
index b5c19f3545c6..8c1e7b1f18ed 100644
--- a/src/main/java/com/google/gcloud/datastore/DoubleValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.DOUBLE_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java
index 292291792442..95d96ac8030a 100644
--- a/src/main/java/com/google/gcloud/datastore/Entity.java
+++ b/src/main/java/com/google/gcloud/datastore/Entity.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java
index c4e46bebc8ae..cdf24d3ca8ed 100644
--- a/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.ENTITY_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 9c5c982b82c4..7ab0831c7e30 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index 43ea51d83c1c..97559478293c 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index 08d854404a3b..90be65882a44 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java
index 65163103ede4..959deb0831d4 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyValue.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.KEY_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java
index 4132858432b4..abf7ff9c9a40 100644
--- a/src/main/java/com/google/gcloud/datastore/ListValue.java
+++ b/src/main/java/com/google/gcloud/datastore/ListValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.LIST_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/src/main/java/com/google/gcloud/datastore/LongValue.java
index ac887e4c7263..e69d87d05e04 100644
--- a/src/main/java/com/google/gcloud/datastore/LongValue.java
+++ b/src/main/java/com/google/gcloud/datastore/LongValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.INTEGER_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java
index 58c91e8342cb..77c813c9068b 100644
--- a/src/main/java/com/google/gcloud/datastore/NullValue.java
+++ b/src/main/java/com/google/gcloud/datastore/NullValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkArgument;
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index b774e30c2a3d..1831d810e831 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/PartialKey.java
index 74659534b3c1..ae4c706f76c7 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialKey.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/main/java/com/google/gcloud/datastore/PathElement.java b/src/main/java/com/google/gcloud/datastore/PathElement.java
index b3ec8b55e5d3..186ed97adcde 100644
--- a/src/main/java/com/google/gcloud/datastore/PathElement.java
+++ b/src/main/java/com/google/gcloud/datastore/PathElement.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkArgument;
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
index 22201f82dda4..7d62ee88405f 100644
--- a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 17084249c646..a8f4c97fdc72 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index 95fb993b1771..65d087d4b4a9 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import java.util.Iterator;
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
index f1f6ee2f4838..2944724e116b 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/main/java/com/google/gcloud/datastore/RawValue.java b/src/main/java/com/google/gcloud/datastore/RawValue.java
index 17e5b70b12ab..b2e8b9785dd8 100644
--- a/src/main/java/com/google/gcloud/datastore/RawValue.java
+++ b/src/main/java/com/google/gcloud/datastore/RawValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/main/java/com/google/gcloud/datastore/Serializable.java b/src/main/java/com/google/gcloud/datastore/Serializable.java
index 3770ecfccf7f..ff62fe89195f 100644
--- a/src/main/java/com/google/gcloud/datastore/Serializable.java
+++ b/src/main/java/com/google/gcloud/datastore/Serializable.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.protobuf.GeneratedMessage;
diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java
index ae09e39b6b6e..699d2bd97cd7 100644
--- a/src/main/java/com/google/gcloud/datastore/StringValue.java
+++ b/src/main/java/com/google/gcloud/datastore/StringValue.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER;
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 671e7403b511..d86086f53088 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.api.client.util.Preconditions.checkNotNull;
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index fea662ead76a..099248465587 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import java.util.Iterator;
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 1b33ae19b4d8..86f5a8908a22 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest;
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
index 433787a02f69..c1c8368213dc 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionOption.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionOption.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/src/main/java/com/google/gcloud/datastore/Validator.java
index d2586573d9e8..9ee6a5e57d68 100644
--- a/src/main/java/com/google/gcloud/datastore/Validator.java
+++ b/src/main/java/com/google/gcloud/datastore/Validator.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkArgument;
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index ff064e87178e..2db172039535 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static com.google.common.base.Preconditions.checkArgument;
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index 88933a598ca9..7661deebb82c 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 /**
  * A client to the Google Cloud Datastore.
  *
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index a0ef42844b7d..4c218848ea9a 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 public interface Acl {
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index be2653494fd9..5fcd7f4abe50 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 import java.nio.ByteBuffer;
diff --git a/src/main/java/com/google/gcloud/storage/Key.java b/src/main/java/com/google/gcloud/storage/Key.java
index 759cc5e9496f..1e16d985c021 100644
--- a/src/main/java/com/google/gcloud/storage/Key.java
+++ b/src/main/java/com/google/gcloud/storage/Key.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 public class Key {
diff --git a/src/main/java/com/google/gcloud/storage/StorageObject.java b/src/main/java/com/google/gcloud/storage/StorageObject.java
index 0fbee66031b8..0bbdd3b20cc1 100644
--- a/src/main/java/com/google/gcloud/storage/StorageObject.java
+++ b/src/main/java/com/google/gcloud/storage/StorageObject.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 import java.nio.ByteBuffer;
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 47e2af985f3a..55b9b84c1e2b 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 public interface StorageService {
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
index acc243b9ba38..7f94334b3c08 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 85c57bea3a9b..45a347d27c6d 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 import com.google.api.services.storage.Storage;
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 6cdcdc65bf0c..a4a8e60adf94 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.storage;
 
 import com.google.api.client.json.jackson.JacksonFactory;
diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/src/main/java/com/google/gcloud/storage/package-info.java
index 0f78d1bab46a..9aabec93c93f 100644
--- a/src/main/java/com/google/gcloud/storage/package-info.java
+++ b/src/main/java/com/google/gcloud/storage/package-info.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 /**
  * A client to Google Cloud Storage.
  *
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
index 960f293e0baa..1d96e86d0e46 100644
--- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud;
 
 import static org.junit.Assert.assertFalse;
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/src/test/java/com/google/gcloud/RetryHelperTest.java
index b8b0f7a4c501..dfd933bcae46 100644
--- a/src/test/java/com/google/gcloud/RetryHelperTest.java
+++ b/src/test/java/com/google/gcloud/RetryHelperTest.java
@@ -1,11 +1,11 @@
 /*
- * Copyright 2012 Google Inc. All Rights Reserved.
+ * Copyright 2015 Google Inc. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *       http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/src/test/java/com/google/gcloud/RetryParamsTest.java b/src/test/java/com/google/gcloud/RetryParamsTest.java
index 744bec7b59a2..d1d5e3c076d8 100644
--- a/src/test/java/com/google/gcloud/RetryParamsTest.java
+++ b/src/test/java/com/google/gcloud/RetryParamsTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud;
 
 import static com.google.gcloud.RetryParams.DEFAULT_INITIAL_RETRY_DELAY_MILLIS;
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
index 4af76c5ca07e..8bc31d5381c3 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static org.junit.Assert.assertEquals;
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 279cb0b2392e..5a71442b8a6b 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 44fb398bdfb6..89b7b01aadb3 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static java.nio.charset.StandardCharsets.UTF_8;

From 33acc849dc3d5caf3ac52d79791dee8bb6af74ab Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 20 Jan 2015 15:27:56 -0800
Subject: [PATCH 094/732] adding contributing

---
 CONTRIBUTING.md | 50 +++++++++++++++++++++++++++++++++++++++++++++++++
 README.md       |  2 +-
 2 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 CONTRIBUTING.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000000..df4129b3ed6d
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,50 @@
+Contributing
+============
+
+1. **Please sign one of the contributor license agreements below.**
+1. Fork the repo, develop and test your code changes, add docs.
+1. Make sure that your commit messages clearly describe the changes.
+1. Send a pull request.
+
+
+Here are some guidelines for hacking on gcloud-java.
+
+
+Using maven for build/test
+--------------------------
+After you cloned the repository use Maven for building and running the tests.
+Maven 3.0+ is required.
+
+
+Adding Features
+---------------
+In order to add a feature to gcloud-java:
+
+The feature must be fully documented using Javadoc and examples should be provided.
+The feature must work fully on Java 7 and above.
+The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective,
+but new dependencies should be discussed).
+
+
+Coding Style
+------------
+Maintain the coding style in the project and in particular the modified files.
+Follow the Google Java [style](http://google-styleguide.googlecode.com/svn/trunk/javaguide.html).
+
+
+## Contributor License Agreements
+
+Before we can accept your pull requests you'll need to sign a Contributor
+License Agreement (CLA):
+
+- **If you are an individual writing original source code** and **you own the intellectual property**,
+then you'll need to sign an [individual CLA][indvcla].
+- **If you work for a company that wants to allow you to contribute your work**,
+then you'll need to sign a [corporate CLA][corpcla].
+
+You can sign these electronically (just scroll to the bottom). After that,
+we'll be able to accept your pull requests.
+
+[gcloudcli]: https://developers.google.com/cloud/sdk/gcloud/
+[indvcla]: https://developers.google.com/open-source/cla/individual
+[corpcla]: https://developers.google.com/open-source/cla/corporate
diff --git a/README.md b/README.md
index 3b9e93c1492b..bf88957bae80 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,7 @@ Java idiomatic client for Google Cloud Platform services. Supported APIs include
 > Note: This package is a work-in-progress, and may occasionally
 > make backwards-incompatible changes.
 
-Documentation and examples are available [here](https://github.com/GoogleCloudePlatform/gcloud-java/gh-pages/docs).
+Documentation and examples are available [here](https://github.com/GoogleCloudePlatform/gcloud-java/gh-pages/apidocs).
 
 ## Google Cloud Datastore
 

From cb0709d0ecf4b781d05a87116b727c8e95cb91c9 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 20 Jan 2015 15:53:46 -0800
Subject: [PATCH 095/732] adding travis config file

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 000000000000..b2168ad38271
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,5 @@
+language: java
+jdk:
+  - oraclejdk8
+  - oraclejdk7
+  - openjdk7

From 30eb51cbf99c58c3d157d59c867326207c108636 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 20 Jan 2015 16:04:37 -0800
Subject: [PATCH 096/732] Make travis run verify instead of test

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index b2168ad38271..1b9769d8b85e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,3 +3,5 @@ jdk:
   - oraclejdk8
   - oraclejdk7
   - openjdk7
+install: mvn install -DskipTests=true
+script: mvn verify

From 9c1f46180c8a38760c160886cb3e57a0bca2c9e5 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 20 Jan 2015 16:28:44 -0800
Subject: [PATCH 097/732] change maven group information

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 69c2673fe060..a45df9dff031 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2,8 +2,8 @@
 
   4.0.0
-  com.ozarov.testing
-  git-demo
+  com.google.gcloud
+  gcloud-java
   0.0.1-SNAPSHOT
   
     

From e0113f30b6d767bff72257b8f6f821d4a5149caa Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 9 Feb 2015 15:29:58 -0800
Subject: [PATCH 098/732] adding travis credentials

---
 .travis.yml | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1b9769d8b85e..5997f22ead37 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,21 @@
-language: java
+nguage: java
 jdk:
-  - oraclejdk8
-  - oraclejdk7
-  - openjdk7
+- oraclejdk8
+- oraclejdk7
+- openjdk7
+before_install:
+- mvn clean
+- git clone -b travis `git config --get remote.origin.url` target/travis
+- cp target/travis/settings.xml ~/.m2/settings.xml
 install: mvn install -DskipTests=true
 script: mvn verify
+branches:
+  only:
+  - master
+after_success:
+- mvn cobertura:cobertura coveralls:report
+- mvn site --settings target/travis/settings.xml
+env:
+  global:
+  - secure: bjyc4GJSP9850m6KSO2LiGKMJI/iFJ6dIDNrrZJHiokWUv8ID5+X7O04YtAFF+WrYyVDJ8Zs+uduAJaQ5NFesnhFjMMNTOaliYIBjpBgdZU0vgmsU0NzO35bu6wA5DAdI8AGUNCVwSZpOAMnj/80dbYbyFwBn2DWBZ3QwpV6J/I=
+  - secure: CUM2l73KFm7U4eDsUKkh1WyEUzF3v94Ltvs7MnKU9olE1dNp3YmRBL9Lqhx3hSDqm/xv0ETQsPy29Fs2+VFkhQQxSley6iS/4trr2fioTB680txfXo/zDdmGSP1q1/U40fv1S+jvuBRAhDV5W+8dhWOGtzMH0tJp/TszeDGlmCY=

From 72551d224965f1ab92bc6f46c582884b4849cfa1 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 9 Feb 2015 15:41:56 -0800
Subject: [PATCH 099/732] apply changes from git-demo

---
 README.md                                     |   3 +-
 checkstyle.xml                                | 363 ++++++++++--------
 pom.xml                                       | 217 ++++++++---
 .../datastore/BaseDatastoreBatchWriter.java   | 169 ++++++++
 .../com/google/gcloud/datastore/Batch.java    |   5 +-
 .../google/gcloud/datastore/BatchImpl.java    | 159 +-------
 .../datastore/DatastoreBatchWriter.java       |   6 -
 .../google/gcloud/datastore/Transaction.java  |   4 +-
 .../gcloud/datastore/TransactionImpl.java     |  89 ++---
 .../com/google/gcloud/storage/Bucket.java     |  26 +-
 .../storage/{Key.java => InputChannel.java}   |  33 +-
 .../google/gcloud/storage/OutputChannel.java  |  33 ++
 .../google/gcloud/storage/StorageObject.java  |  65 +++-
 .../google/gcloud/storage/StorageService.java |   2 +-
 src/site/apt/index.apt                        |  23 ++
 src/site/site.xml                             |  11 +
 ...ionTest.java => DatastoreServiceTest.java} |  21 +-
 17 files changed, 764 insertions(+), 465 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
 rename src/main/java/com/google/gcloud/storage/{Key.java => InputChannel.java} (57%)
 create mode 100644 src/main/java/com/google/gcloud/storage/OutputChannel.java
 create mode 100644 src/site/apt/index.apt
 create mode 100644 src/site/site.xml
 rename src/test/java/com/google/gcloud/datastore/{DatastoreServiceIntegrationTest.java => DatastoreServiceTest.java} (98%)

diff --git a/README.md b/README.md
index bf88957bae80..7e9c47ebd268 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@ Google Cloud for Java
 =====================
 
 [![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
+[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java)
 
 Java idiomatic client for Google Cloud Platform services. Supported APIs include:
 
@@ -11,7 +12,7 @@ Java idiomatic client for Google Cloud Platform services. Supported APIs include
 > Note: This package is a work-in-progress, and may occasionally
 > make backwards-incompatible changes.
 
-Documentation and examples are available [here](https://github.com/GoogleCloudePlatform/gcloud-java/gh-pages/apidocs).
+Documentation and examples are available [here](https://googlecloudeplatform.github.com/gcloud-java/apidocs).
 
 ## Google Cloud Datastore
 
diff --git a/checkstyle.xml b/checkstyle.xml
index 814915090b25..7b6d2abd5db8 100644
--- a/checkstyle.xml
+++ b/checkstyle.xml
@@ -1,171 +1,200 @@
-
-
+
+
 
 
-
-
-  
-  
-  
-    
-    
-      
-      
-      
-    
-    
-      
-      
-      
-    
-    
-      
-      
-    
-    
-    
-    
-    
-      
-      
-    
-    
-    
-      
-    
-    
-    
-      
-      
-    
-    
-      
-      
-      
-      
-      
-      
-    
-    
-    
-    
-    
-    
-    
-    
-    
-      
-    
-    
-      
-      
-    
-    
-      
-      
-    
-    
-      
-      
-    
-    
-      
-    
-    
-      
-      
-    
-    
-      
-      
-    
-    
-      
-      
-      
-      
-    
-    
-      
-      
-    
-    
-      
-      
-    
-    
-    
-      
-      
-      
-      
-    
-    
-      
-      
-      
-    
-    
-      
-      
-    
-    
-    
-    
-      
-      
-      
-      
-      
-      
-    
-    
-    
-      
-      
-    
-    
-      
-    
-    
-      
-      
-    
-    
-    
-    
-    
-    
-      
-      
-    
-    
-      
-      
-      
-      
-      
-      
-      
-    
-    
-      
-      
-    
-    
-    
-      
-      
-      
+
+    Checkstyle configuration that checks the Google coding conventions from:
+    
+    -  Google Java Style
+       https://google-styleguide.googlecode.com/svn-history/r130/trunk/javaguide.html
+       
+    Checkstyle is very configurable. Be sure to read the documentation at
+    http://checkstyle.sf.net (or in your downloaded distribution).
+
+    Most Checks are configurable, be sure to consult the documentation.
+
+    To completely disable a check, just comment it out or delete it from the file.
+
+    Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
+    
+ -->
+ 
+
+    
+    
+    
+    
+    
+    
+        
+            
+        
+    
+    
+        
+        
+            
+            
+            
+        
+        
+            
+            
+            
+        
+        
+            
+            
+        
+        
+        
+        
+        
+            
+            
+        
+        
+        
+            
+        
+        
+        
+            
+            
+        
+        
+            
+            
+            
+            
+            
+             
+        
+        
+        
+        
+        
+        
+        
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+        
+            
+             
+             
+             
+        
+        
+            
+            
+            
+            
+            
+            
+        
+        
+            
+            
+        
+        
+        
+        
+            
+            
+            
+            
+        
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+        
+        
+            
+        
+        
+        
+            
+            
+        
+        
+            
+            
+            
+            
+            
+            
+            
+        
+        
+            
+            
+        
+        
     
-  
-  
-    
-  
 
diff --git a/pom.xml b/pom.xml
index a45df9dff031..f9a5dc52f8b9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,10 +1,29 @@
 
-
+
   4.0.0
   com.google.gcloud
   gcloud-java
+  jar
   0.0.1-SNAPSHOT
+  GCloud Java
+  https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    Java idiomatic client for Google Cloud Platform services.
+  
+  
+    scm:git:git@github.com:GoogleCloudPlatform/gcloud-java.git
+    scm:git:git@github.com:GoogleCloudPlatform/gcloud-java.git
+    https://github.com/GoogleCloudPlatform/gcloud-java
+    HEAD
+  
+  
+    Travis CI
+    https://travis-ci.org/GoogleCloudPlatform/gcloud-java
+  
+  
+    https://github.com/GoogleCloudPlatform/gcloud-java/issues
+    GitHub Issues
+  
   
     
       com.google.http-client
@@ -87,9 +106,16 @@
       http://repo.maven.apache.org/maven2
     
   
+  
+    
+      GCloud Java Software License
+      https://raw.github.com/GoogleCloudPlatform/gcloud-java/master/LICENSE.md
+    
+  
   
     UTF-8
     UTF-8
+    github
   
   
     
@@ -106,70 +132,10 @@
   
   
     
-      
-        maven-antrun-plugin
-        1.8
-        
-          
-            before-integration-test
-            pre-integration-test
-            
-              
-                
-                
-                  
-                  
-                    
-                  
-                
-              
-            
-            
-              run
-            
-          
-          
-            after-integration-test
-            post-integration-test
-            
-              
-                
-                
-                  
-                  
-                    
-                  
-                
-              
-            
-            
-              run
-            
-          
-        
-      
-      
-        org.apache.maven.plugins
-        maven-surefire-plugin
-        2.18.1
-        
-          
-            **/*IntegrationTest.java
-          
-        
-      
       
         org.apache.maven.plugins
         maven-failsafe-plugin
         2.18.1
-        
-          
-            none
-          
-          
-            **/*IntegrationTest.java
-          
-        
         
           
             
@@ -192,6 +158,133 @@
           UTF-8
         
       
+      
+        org.eluder.coveralls
+        coveralls-maven-plugin
+        3.0.1
+      
+      
+        org.codehaus.mojo
+        cobertura-maven-plugin
+        2.6
+        
+          
+            xml
+            html
+          
+          true
+          
+            true
+            
+              com/google/gcloud/**/*.class
+            
+          
+          
+          256m
+        
+      
+      
+        org.apache.maven.plugins
+        maven-checkstyle-plugin
+        2.13
+        
+          
+            com.puppycrawl.tools
+            checkstyle
+            6.2
+          
+        
+      
+      
+        org.apache.maven.plugins
+        maven-site-plugin
+        3.4
+        
+          
+            
+	      org.apache.maven.plugins
+              maven-changelog-plugin
+              2.3
+            
+            
+              org.apache.maven.plugins
+              maven-project-info-reports-plugin
+              2.8
+              
+	        
+		  
+		    dependencies
+		    project-team
+		    mailing-list
+              	    cim
+		    issue-tracking
+		    license
+		    scm
+                  
+                
+              
+	      
+                true
+                true
+                true
+              
+            
+            
+              org.apache.maven.plugins
+              maven-javadoc-plugin
+              2.10.1
+              
+                
+                  html
+                  
+                    javadoc
+                  
+                
+              
+              
+                protected
+                true
+                ${project.build.directory}/javadoc
+              
+            
+            
+              org.apache.maven.plugins
+              maven-surefire-report-plugin
+              2.18.1
+            
+            
+              org.apache.maven.plugins
+              maven-checkstyle-plugin
+              2.13
+              
+                checkstyle.xml
+                false
+              
+            
+            
+              org.codehaus.mojo
+              cobertura-maven-plugin
+              2.6
+            
+          
+        
+      
+      
+        com.github.github
+        site-maven-plugin
+        0.10
+        
+          Creating site for ${project.artifactId} ${project.version}
+        
+        
+          
+            
+              site
+            
+            site
+          
+        
+      
     
   
 
diff --git a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
new file mode 100644
index 000000000000..f2f60ef5d30f
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import com.google.api.services.datastore.DatastoreV1;
+
+import java.util.*;
+
+/**
+ * Base class for DatastoreBatchWriter.
+ */
+public abstract class BaseDatastoreBatchWriter implements DatastoreBatchWriter {
+
+  private final String name;
+  private final Map toAdd = new LinkedHashMap<>();
+  private final List toAddAutoId = new LinkedList<>();
+  private final Map toUpdate = new LinkedHashMap<>();
+  private final Map toPut = new LinkedHashMap<>();
+  private final Set toDelete = new LinkedHashSet<>();
+  private boolean active = true;
+
+  protected BaseDatastoreBatchWriter(String name) {
+    this.name = name;
+  }
+
+  @Override
+  public void add(Entity... entities) {
+    validateActive();
+    for (Entity entity : entities) {
+      Key key = entity.key();
+      if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) {
+        throw newInvalidRequest("Entity with the key %s was already added or updated in this %s",
+            entity.key(), name);
+      }
+      if (toDelete.remove(key)) {
+        toPut.put(key, entity);
+      } else {
+        toAdd.put(key, entity);
+      }
+    }
+  }
+
+  @Override
+  public void add(PartialEntity... entities) {
+    validateActive();
+    for (PartialEntity entity : entities) {
+      if (entity instanceof Entity) {
+        add((Entity) entity);
+      } else {
+        toAddAutoId.add(entity);
+      }
+    }
+  }
+
+  @Override
+  public void update(Entity... entities) {
+    validateActive();
+    for (Entity entity : entities) {
+      Key key = entity.key();
+      if (toDelete.contains(key)) {
+        throw newInvalidRequest("Entity with the key %s was already deleted in this %s",
+            entity.key(), name);
+      }
+      if (toAdd.remove(key) != null || toPut.containsKey(key)) {
+        toPut.put(key, entity);
+      } else {
+        toUpdate.put(key, entity);
+      }
+    }
+  }
+
+  @Override
+  public void put(Entity... entities) {
+    validateActive();
+    for (Entity entity : entities) {
+      Key key = entity.key();
+      toAdd.remove(key);
+      toUpdate.remove(key);
+      toDelete.remove(key);
+      toPut.put(key, entity);
+    }
+  }
+
+  @Override
+  public void delete(Key... keys) {
+    validateActive();
+    for (Key key : keys) {
+      toAdd.remove(key);
+      toUpdate.remove(key);
+      toPut.remove(key);
+      toDelete.add(key);
+    }
+  }
+
+  @Override
+  public boolean active() {
+    return active;
+  }
+
+  protected String name() {
+    return name;
+  }
+
+  protected Map toAdd() {
+    return toAdd;
+  }
+
+  protected List toAddAutoId() {
+    return toAddAutoId;
+  }
+
+  protected Map toUpdate() {
+    return toUpdate;
+  }
+
+  protected Map toPut() {
+    return toPut;
+  }
+
+  protected Set toDelete() {
+    return toDelete;
+  }
+
+  protected void deactivate() {
+    active = false;
+  }
+
+  protected void validateActive() {
+    if (!active) {
+      throw newInvalidRequest("%s is no longer active", name);
+    }
+  }
+
+  protected DatastoreServiceException newInvalidRequest(String msg, Object... params) {
+    return DatastoreServiceException.throwInvalidRequest(String.format(msg, params));
+  }
+
+  protected DatastoreV1.Mutation.Builder toMutationPb() {
+    DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
+    for (PartialEntity entity : toAddAutoId()) {
+      mutationPb.addInsertAutoId(entity.toPb());
+    }
+    for (Entity entity : toAdd().values()) {
+      mutationPb.addInsert(entity.toPb());
+    }
+    for (Entity entity : toUpdate().values()) {
+      mutationPb.addUpdate(entity.toPb());
+    }
+    for (Entity entity : toPut().values()) {
+      mutationPb.addUpsert(entity.toPb());
+    }
+    for (Key key : toDelete()) {
+      mutationPb.addDelete(key.toPb());
+    }
+    return mutationPb;
+  }
+}
diff --git a/src/main/java/com/google/gcloud/datastore/Batch.java b/src/main/java/com/google/gcloud/datastore/Batch.java
index 0a78ad3dff71..a6ba78160ef4 100644
--- a/src/main/java/com/google/gcloud/datastore/Batch.java
+++ b/src/main/java/com/google/gcloud/datastore/Batch.java
@@ -16,6 +16,8 @@
 
 package com.google.gcloud.datastore;
 
+import java.util.List;
+
 /**
  * An interface to represent a batch of write operations.
  * Any write operation that is applied on a batch will only be sent
@@ -34,7 +36,8 @@
  */
 public interface Batch extends DatastoreBatchWriter {
 
-  interface Response extends DatastoreBatchWriter.Response {
+  interface Response {
+    List generatedKeys();
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
index 4f24c2ad77df..7139ca3bb1c6 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -1,47 +1,34 @@
 /*
  * Copyright 2015 Google Inc. All Rights Reserved.
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
  *
- *       http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
  */
 
 package com.google.gcloud.datastore;
 
-import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest;
-
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
 import com.google.gcloud.datastore.BatchOption.ForceWrites;
 
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-class BatchImpl implements Batch {
 
-  private final Map toAdd = new LinkedHashMap<>();
-  private final List toAddAutoId = new LinkedList<>();
-  private final Map toUpdate = new LinkedHashMap<>();
-  private final Map toPut = new LinkedHashMap<>();
-  private final Set toDelete = new LinkedHashSet<>();
+class BatchImpl extends BaseDatastoreBatchWriter implements Batch {
+
+  private final DatastoreServiceImpl datastore;
   private final boolean force;
-  final DatastoreServiceImpl datastore;
-  private boolean active = true;
 
-  static class ResponseImpl implements Response {
+  static class ResponseImpl implements Batch.Response {
 
     private final DatastoreV1.CommitResponse response;
 
@@ -61,9 +48,9 @@ public List generatedKeys() {
   }
 
   BatchImpl(DatastoreServiceImpl datastore, BatchOption... options) {
+    super("batch");
     this.datastore = datastore;
-    Map, BatchOption> optionsMap =
-        BatchOption.asImmutableMap(options);
+    Map, BatchOption> optionsMap = BatchOption.asImmutableMap(options);
     if (optionsMap.containsKey(ForceWrites.class)) {
       force = ((ForceWrites) optionsMap.get(ForceWrites.class)).force();
     } else {
@@ -71,126 +58,18 @@ public List generatedKeys() {
     }
   }
 
-  void validateActive() {
-    if (!active) {
-      throw throwInvalidRequest(getName() + " is no longer active");
-    }
-  }
-
-  String getName() {
-    return "batch";
-  }
-
-  @Override
-  public void add(Entity... entities) {
-    validateActive();
-    for (Entity entity : entities) {
-      Key key = entity.key();
-      if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) {
-        throw throwInvalidRequest("Entity with the key %s was already added or updated in this "
-            + getName(), entity.key());
-      }
-      if (toDelete.remove(key)) {
-        toPut.put(key, entity);
-      } else {
-        toAdd.put(key, entity);
-      }
-    }
-  }
-
-  @Override
-  public void add(PartialEntity... entities) {
-    validateActive();
-    for (PartialEntity entity : entities) {
-      if (entity instanceof Entity) {
-        add((Entity) entity);
-      } else {
-        toAddAutoId.add(entity);
-      }
-    }
-  }
-
-  @Override
-  public void update(Entity... entities) {
-    validateActive();
-    for (Entity entity : entities) {
-      Key key = entity.key();
-      if (toDelete.contains(key)) {
-        throw throwInvalidRequest(
-            "Entity with the key %s was already deleted in this " + getName(), entity.key());
-      }
-      if (toAdd.remove(key) != null || toPut.containsKey(key)) {
-        toPut.put(key, entity);
-      } else {
-        toUpdate.put(key, entity);
-      }
-    }
-  }
-
-  @Override
-  public void put(Entity... entities) {
-    validateActive();
-    for (Entity entity : entities) {
-      Key key = entity.key();
-      toAdd.remove(key);
-      toUpdate.remove(key);
-      toDelete.remove(key);
-      toPut.put(key, entity);
-    }
-  }
-
-  @Override
-  public void delete(Key... keys) {
-    validateActive();
-    for (Key key : keys) {
-      toAdd.remove(key);
-      toUpdate.remove(key);
-      toPut.remove(key);
-      toDelete.add(key);
-    }
-  }
-
   @Override
-  public Response submit() {
-    return new ResponseImpl(commitRequest());
-  }
-
-  DatastoreV1.CommitResponse commitRequest() {
+  public Batch.Response submit() {
     validateActive();
-    DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    for (PartialEntity entity : toAddAutoId) {
-      mutationPb.addInsertAutoId(entity.toPb());
-    }
-    for (Entity entity : toAdd.values()) {
-      mutationPb.addInsert(entity.toPb());
-    }
-    for (Entity entity : toUpdate.values()) {
-      mutationPb.addUpdate(entity.toPb());
-    }
-    for (Entity entity : toPut.values()) {
-      mutationPb.addUpsert(entity.toPb());
-    }
-    for (Key key : toDelete) {
-      mutationPb.addDelete(key.toPb());
-    }
+    DatastoreV1.Mutation.Builder mutationPb = toMutationPb();
     if (force) {
       mutationPb.setForce(force);
     }
-    DatastoreV1.CommitRequest.Builder requestPb = newCommitRequest();
-    requestPb.setMutation(mutationPb);
-    DatastoreV1.CommitResponse response = datastore.commit(requestPb.build());
-    active = false;
-    return response;
-  }
-
-  @Override
-  public boolean active() {
-    return active;
-  }
-
-  DatastoreV1.CommitRequest.Builder newCommitRequest() {
     DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
     requestPb.setMode(DatastoreV1.CommitRequest.Mode.NON_TRANSACTIONAL);
-    return requestPb;
+    requestPb.setMutation(mutationPb);
+    DatastoreV1.CommitResponse responsePb = datastore.commit(requestPb.build());
+    deactivate();
+    return new ResponseImpl(responsePb);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
index 25c2409e58ea..dce06fb56304 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
@@ -16,18 +16,12 @@
 
 package com.google.gcloud.datastore;
 
-import java.util.List;
-
 /**
  * An interface to represent a batch of write operations.
  * All write operation for a batch writer will be applied to the Datastore in one RPC call.
  */
 interface DatastoreBatchWriter extends DatastoreWriter {
 
-  interface Response {
-    List generatedKeys();
-  }
-
   /**
    * {@inheritDoc}
    * This operation will be converted to {@link #put} operation for entities that were already
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index 099248465587..543d82b2d75b 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -17,6 +17,7 @@
 package com.google.gcloud.datastore;
 
 import java.util.Iterator;
+import java.util.List;
 
 /**
  * A Google cloud datastore transaction.
@@ -51,7 +52,8 @@
  */
 public interface Transaction extends DatastoreBatchWriter, DatastoreReaderWriter {
 
-  interface Response extends DatastoreBatchWriter.Response {
+  interface Response {
+    List generatedKeys();
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 86f5a8908a22..bb19e25a2b8f 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -16,32 +16,47 @@
 
 package com.google.gcloud.datastore;
 
-import static com.google.gcloud.datastore.DatastoreServiceException.throwInvalidRequest;
-
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.base.Function;
 import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.gcloud.datastore.TransactionOption.ForceWrites;
 import com.google.gcloud.datastore.TransactionOption.IsolationLevel;
 import com.google.protobuf.ByteString;
 
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-final class TransactionImpl extends BatchImpl implements Transaction {
+final class TransactionImpl extends BaseDatastoreBatchWriter implements Transaction {
 
+  private final DatastoreServiceImpl datastore;
   private final ByteString transaction;
+  private final boolean force;
   private boolean rolledback;
 
-  static class ResponseImpl extends BatchImpl.ResponseImpl implements Transaction.Response {
+  static class ResponseImpl implements Transaction.Response {
+
+    private final DatastoreV1.CommitResponse response;
 
     public ResponseImpl(DatastoreV1.CommitResponse response) {
-      super(response);
+      this.response = response;
+    }
+
+    @Override
+    public List generatedKeys() {
+      return Lists.transform(response.getMutationResult().getInsertAutoIdKeyList(),
+          new Function() {
+            @Override public Key apply(DatastoreV1.Key keyPb) {
+              return Key.fromPb(keyPb);
+            }
+          });
     }
   }
 
   TransactionImpl(DatastoreServiceImpl datastore, TransactionOption... options) {
-    super(datastore, getBatchOptions(options));
+    super("transaction");
+    this.datastore = datastore;
     DatastoreV1.BeginTransactionRequest.Builder requestPb =
         DatastoreV1.BeginTransactionRequest.newBuilder();
     Map, TransactionOption> optionsMap =
@@ -50,20 +65,11 @@ public ResponseImpl(DatastoreV1.CommitResponse response) {
     if (isolationLevel != null) {
       requestPb.setIsolationLevel(isolationLevel.level().toPb());
     }
+    ForceWrites forceWrites = (ForceWrites) optionsMap.get(TransactionOption.ForceWrites.class);
+    force = forceWrites == null ? false : forceWrites.force();
     transaction = datastore.requestTransactionId(requestPb);
   }
 
-  private static BatchOption[] getBatchOptions(TransactionOption... options) {
-    List batchOptions = new ArrayList<>(options.length);
-    for (TransactionOption option : options) {
-      BatchOption batchOption = option.toBatchWriteOption();
-      if (batchOption != null) {
-        batchOptions.add(batchOption);
-      }
-    }
-    return batchOptions.toArray(new BatchOption[batchOptions.size()]);
-  }
-
   @Override
   public Entity get(Key key) {
     return Iterators.getNext(get(new Key[] {key}), null);
@@ -87,41 +93,28 @@ public  QueryResult run(Query query) {
 
   @Override
   public Transaction.Response commit() {
-    return new ResponseImpl(commitRequest());
-  }
-
-  @Override
-  public void rollback() {
-    super.validateActive();
-    if (!rolledback) {
-      datastore.rollbackTransaction(transaction);
+    validateActive();
+    DatastoreV1.Mutation.Builder mutationPb = toMutationPb();
+    if (force) {
+      mutationPb.setForce(force);
     }
-    rolledback = true;
-  }
-
-  @Override
-  public boolean active() {
-    return super.active() && !rolledback;
-  }
-
-  @Override
-  protected String getName() {
-    return "transaction";
+    DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
+    requestPb.setMode(DatastoreV1.CommitRequest.Mode.TRANSACTIONAL);
+    requestPb.setTransaction(transaction);
+    requestPb.setMutation(mutationPb);
+    DatastoreV1.CommitResponse responsePb = datastore.commit(requestPb.build());
+    deactivate();
+    return new ResponseImpl(responsePb);
   }
 
   @Override
-  protected void validateActive() {
-    super.validateActive();
+  public void rollback() {
     if (rolledback) {
-      throw throwInvalidRequest(getName() + " is not active (was rolledback)");
+      return;
     }
-  }
-
-  @Override
-  protected DatastoreV1.CommitRequest.Builder newCommitRequest() {
-    DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
-    requestPb.setMode(DatastoreV1.CommitRequest.Mode.TRANSACTIONAL);
-    requestPb.setTransaction(transaction);
-    return requestPb;
+    validateActive();
+    datastore.rollbackTransaction(transaction);
+    deactivate();
+    rolledback = true;
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 5fcd7f4abe50..86942b1779e2 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -16,7 +16,7 @@
 
 package com.google.gcloud.storage;
 
-import java.nio.ByteBuffer;
+import java.util.Iterator;
 
 public interface Bucket {
 
@@ -24,33 +24,39 @@ public interface Bucket {
 
   String name();
 
+  Cors cors();
+
   Acl acl();
 
   Acl defaultObjectAcl();
 
-  Cors cors();
+  void updateCors(Cors cors);
 
-  void updateDefaultObjectAcl();
   void updateAcl(Acl acl);
 
+  void updateDefaultObjectAcl();
 
+  void delete(String... objectName);
 
+  void compose(Iterable sourceObjectNames, String destObjectName);
 
+  void copy(String sourceObjectName, StorageObject.Key destObjectKey);
 
 
-  void delete(Key... objectKey);
-
-  void compose(Iterable source, String dest);
-
-  void copy(String source, String dest);
-
   // TODO (ozarov): consider replace with Object that has a reference to bucket and name
   // that object can return its own meta-data, update its own meta-data, replace its content
   // via a stream or byteBuffer, read its content (via stream or ByteBuffer),...
   //void copy(String source, String bucket, String dest);
   // Also consider read with an offset (and limit).
 
-  void put(String name, ByteBuffer bytes);
+  // returns null if not exists
+  StorageObject get(String objectName);
+
+  Iterator get(String... objectName);
+
+  InputChannel getInputChannel(String ObjectName);
+
+  OutputChannel getOutputChannel(String ObjectName);
 
   // TODO: add listing
 }
diff --git a/src/main/java/com/google/gcloud/storage/Key.java b/src/main/java/com/google/gcloud/storage/InputChannel.java
similarity index 57%
rename from src/main/java/com/google/gcloud/storage/Key.java
rename to src/main/java/com/google/gcloud/storage/InputChannel.java
index 1e16d985c021..901ccf048ee9 100644
--- a/src/main/java/com/google/gcloud/storage/Key.java
+++ b/src/main/java/com/google/gcloud/storage/InputChannel.java
@@ -16,27 +16,22 @@
 
 package com.google.gcloud.storage;
 
-public class Key {
+import java.io.Closeable;
+import java.io.Serializable;
+import java.nio.channels.ReadableByteChannel;
 
-  // TODO: add builder, factory method, toURL, from URL, equals,hashCode, toString
-  private final String bucket;
-  private final String name;
-
-  /*
-  Builder() {
-
-  }*/
+/**
+ * A readable byte channel for reading data from Google Cloud Storage.
+ *
+ *  Implementations of this class may buffer data internally to reduce remote calls.
+ *
+ *  This class is @{link Serializable}, which allows incremental reads.
+ */
+public interface InputChannel extends ReadableByteChannel, Serializable, Closeable {
 
-  Key(String bucket, String name) {
-    this.bucket = bucket;
-    this.name = name;
-  }
+  StorageObject.Key key();
 
-  public String bucket() {
-    return bucket;
-  }
+  int size();
 
-  public String name() {
-    return name;
-  }
+  void seek(int position);
 }
diff --git a/src/main/java/com/google/gcloud/storage/OutputChannel.java b/src/main/java/com/google/gcloud/storage/OutputChannel.java
new file mode 100644
index 000000000000..cf45e292ce2c
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/OutputChannel.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import java.io.Closeable;
+import java.io.Serializable;
+import java.nio.channels.WritableByteChannel;
+
+/**
+ * A writable byte channel for writing data to Google Cloud Storage.
+ *
+ * Implementations of this class may further buffer data internally to reduce remote calls.
+ * Written data will only be visible after calling {@link #close()}.
+ * This class is serializable, to allow incremental writes.
+ */
+public interface OutputChannel extends WritableByteChannel, Serializable, Closeable {
+
+  StorageObject.Key key();
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageObject.java b/src/main/java/com/google/gcloud/storage/StorageObject.java
index 0bbdd3b20cc1..2a8dcac2f916 100644
--- a/src/main/java/com/google/gcloud/storage/StorageObject.java
+++ b/src/main/java/com/google/gcloud/storage/StorageObject.java
@@ -18,9 +18,65 @@
 
 import java.nio.ByteBuffer;
 
+// TODO: add  equals,hashCode, toString, serializable
 public interface StorageObject {
 
-  // builder will have an option to populate content and set acl, bucket, name,..
+  class Key {
+
+    // TODO: add builder, factory method, toURL, from URL, equals,hashCode, toString, serializable
+    private final String bucket;
+    private final String name;
+
+
+    Key(Bucket bucket, String name) {
+      this.bucket = bucket.name();
+      this.name = name;
+    }
+
+    public String bucket() {
+      return bucket;
+    }
+
+    public String name() {
+      return name;
+    }
+  }
+
+  abstract class Builder {
+
+    private Bucket bucket;
+    private Acl acl;
+    private String name;
+    private ByteBuffer content;
+
+    public Builder() {
+
+    }
+
+    public Builder name(String name) {
+      this.name = name;
+      return this;
+    }
+
+    public Builder bucket(Bucket bucket) {
+      this.bucket = bucket;
+      return this;
+    }
+
+    public Builder acl(Acl acl) {
+      this.acl = acl;
+      return this;
+    }
+
+    public Builder content(ByteBuffer content) {
+      this.content = content;
+      return this;
+    }
+
+    public abstract StorageObject build();
+  }
+
+  boolean exists();
 
   Key key();
 
@@ -28,4 +84,11 @@ public interface StorageObject {
 
   ByteBuffer content();
 
+  void save();
+
+  void delete();
+
+  InputChannel getInputChannel();
+
+  OutputChannel getOutputChannel();
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 55b9b84c1e2b..313a01f82258 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -20,5 +20,5 @@ public interface StorageService {
 
   Iterable listBuckets();
 
-  Bucket getBucket(String bucket);
+  Bucket getBucket(String bucketName);
 }
diff --git a/src/site/apt/index.apt b/src/site/apt/index.apt
new file mode 100644
index 000000000000..01347874fd16
--- /dev/null
+++ b/src/site/apt/index.apt
@@ -0,0 +1,23 @@
+GCloud Java: Idiomatic Java Client for Google Cloud Platform services. 
+
+ This is a Java Client for accessing Google Cloud Platorm services such as Datastore, Storage, PubSub and others.
+ This library is in a early stage of its development and may occasionally make backwards-incompatible changes,
+ but it is already usable.
+ 
+* Features
+ 
+	* {{{https://cloud.google.com/datastore/}Google Cloud Datastore}}
+ 	
+* Links
+  
+   	* {{{https://github.com/aozarov/git-demo}GitHub repository}}
+   	
+   	* {{{http://aozarov.github.io/git-demo/apidocs/index.html}Javadocs}}
+   	
+   	* {{{http://aozarov.github.io/git-demo/dependencies.html}Dependencies}}
+   	
+   	* {{{https://travis-ci.org/aozarov/git-demo} Continous Integration System (Travis-CI)}}
+   	
+   	* {{{https://github.com/aozarov/git-demo/issues?page=1&state=open}Issues}}
+
+   	* {{{https://coveralls.io/r/aozarov/git-demo?branch=master}Coverage}}
diff --git a/src/site/site.xml b/src/site/site.xml
new file mode 100644
index 000000000000..afe6c29f6042
--- /dev/null
+++ b/src/site/site.xml
@@ -0,0 +1,11 @@
+
+  
+    org.apache.maven.skins
+    maven-fluido-skin
+    1.3.1
+  
+  
+    
+  
+
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
similarity index 98%
rename from src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
rename to src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 8bc31d5381c3..10537734ef16 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceIntegrationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -29,9 +29,9 @@
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
-
-import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -42,7 +42,7 @@
 import java.util.List;
 
 @RunWith(JUnit4.class)
-public class DatastoreServiceIntegrationTest {
+public class DatastoreServiceTest {
 
   private static final String DATASET = LocalGcdHelper.DEFAULT_DATASET;
   private static final String KIND1 = "kind1";
@@ -86,13 +86,18 @@ public class DatastoreServiceIntegrationTest {
   private DatastoreServiceOptions options;
   private DatastoreService datastore;
   private DatastoreHelper helper;
-  private LocalGcdHelper gcdHelper;
 
-  @Before
-  public void setUp() throws IOException, InterruptedException {
+  private static LocalGcdHelper gcdHelper;
+
+  @BeforeClass
+  public static void beforeClass() throws IOException, InterruptedException {
     if (!LocalGcdHelper.isActive(DATASET)) {
       gcdHelper = LocalGcdHelper.start(DATASET);
     }
+  }
+
+  @Before
+  public void setUp() throws IOException, InterruptedException {
     options = DatastoreServiceOptions.builder()
         .dataset(DATASET)
         .host("http://localhost:" + LocalGcdHelper.PORT)
@@ -104,8 +109,8 @@ public void setUp() throws IOException, InterruptedException {
     datastore.add(ENTITY1, ENTITY2);
   }
 
-  @After
-  public void tearDown() throws IOException, InterruptedException {
+  @AfterClass
+  public static void afterClass() throws IOException, InterruptedException {
     if (gcdHelper != null) {
       gcdHelper.stop();
     }

From dd656b6ed809b0e16c4669cd5ae92f481c30f591 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 9 Feb 2015 16:10:11 -0800
Subject: [PATCH 100/732] return the datastore service from Batch and
 Transation

---
 src/main/java/com/google/gcloud/datastore/Batch.java         | 5 +++++
 src/main/java/com/google/gcloud/datastore/BatchImpl.java     | 5 +++++
 src/main/java/com/google/gcloud/datastore/Transaction.java   | 5 +++++
 .../java/com/google/gcloud/datastore/TransactionImpl.java    | 5 +++++
 4 files changed, 20 insertions(+)

diff --git a/src/main/java/com/google/gcloud/datastore/Batch.java b/src/main/java/com/google/gcloud/datastore/Batch.java
index a6ba78160ef4..f74ccc288808 100644
--- a/src/main/java/com/google/gcloud/datastore/Batch.java
+++ b/src/main/java/com/google/gcloud/datastore/Batch.java
@@ -46,4 +46,9 @@ interface Response {
    * @throws DatastoreServiceException if there was any failure or if batch is not longer active
    */
   Response submit();
+
+  /**
+   * Returns the batch associated {@link DatastoreService}.
+   */
+  DatastoreService datastore();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
index 7139ca3bb1c6..305bf4fbcf9f 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -72,4 +72,9 @@ public Batch.Response submit() {
     deactivate();
     return new ResponseImpl(responsePb);
   }
+
+  @Override
+  public DatastoreService datastore() {
+    return datastore;
+  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index 543d82b2d75b..28592552787b 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -109,4 +109,9 @@ interface Response {
    */
   @Override
   boolean active();
+
+  /**
+   * Returns the transaction associated {@link DatastoreService}.
+   */
+  DatastoreService datastore();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index bb19e25a2b8f..859f637d4c72 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -117,4 +117,9 @@ public void rollback() {
     deactivate();
     rolledback = true;
   }
+
+  @Override
+  public DatastoreService datastore() {
+    return datastore;
+  }
 }

From d7a50f43bf96c1b9d2d4209acb62cf6a89bfd93a Mon Sep 17 00:00:00 2001
From: Arie 
Date: Mon, 9 Feb 2015 16:19:03 -0800
Subject: [PATCH 101/732] Delete .project

---
 .project | 65 --------------------------------------------------------
 1 file changed, 65 deletions(-)
 delete mode 100644 .project

diff --git a/.project b/.project
deleted file mode 100644
index 89b7470af643..000000000000
--- a/.project
+++ /dev/null
@@ -1,65 +0,0 @@
-
-
-	git-demo
-	
-	
-	
-	
-		
-			org.eclipse.wst.common.project.facet.core.builder
-			
-			
-		
-		
-			org.eclipse.jdt.core.javabuilder
-			
-			
-		
-		
-			net.sf.eclipsecs.core.CheckstyleBuilder
-			
-			
-		
-		
-			edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder
-			
-			
-		
-		
-			edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder
-			
-			
-		
-		
-			org.eclipse.wst.validation.validationbuilder
-			
-			
-		
-		
-			ntut.csie.rleht.builder.RLBuilder
-			
-			
-		
-		
-			ch.acanda.eclipse.pmd.builder.PMDBuilder
-			
-			
-		
-		
-			org.eclipse.m2e.core.maven2Builder
-			
-			
-		
-	
-	
-		org.eclipse.jem.workbench.JavaEMFNature
-		org.eclipse.wst.common.modulecore.ModuleCoreNature
-		org.eclipse.jdt.core.javanature
-		org.eclipse.m2e.core.maven2Nature
-		net.sf.eclipsecs.core.CheckstyleNature
-		edu.umd.cs.findbugs.plugin.eclipse.findbugsNature
-		org.eclipse.wst.common.project.facet.core.nature
-		ntut.csie.rleht.builder.RLNature
-		ch.acanda.eclipse.pmd.builder.PMDNature
-	
-

From 3048007abede192582b1ffdf36779306f42ed60a Mon Sep 17 00:00:00 2001
From: Arie 
Date: Mon, 9 Feb 2015 16:19:10 -0800
Subject: [PATCH 102/732] Delete .classpath

---
 .classpath | 38 --------------------------------------
 1 file changed, 38 deletions(-)
 delete mode 100644 .classpath

diff --git a/.classpath b/.classpath
deleted file mode 100644
index dc5384546f80..000000000000
--- a/.classpath
+++ /dev/null
@@ -1,38 +0,0 @@
-
-
-	
-		
-			
-		
-	
-	
-		
-			
-			
-		
-	
-	
-		
-			
-			
-		
-	
-	
-		
-			
-			
-		
-	
-	
-	
-		
-			
-		
-	
-	
-		
-			
-		
-	
-	
-

From a87985ee4967bcf6436a85edc4be3420d3468f60 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Mon, 9 Feb 2015 16:31:29 -0800
Subject: [PATCH 103/732] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7e9c47ebd268..7bbb1da1f68d 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@ Google Cloud for Java
 =====================
 
 [![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
-[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java)
+[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master)
 
 Java idiomatic client for Google Cloud Platform services. Supported APIs include:
 

From a2f9386e65674984f0889b4e3d2331842a03d8a7 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Mon, 9 Feb 2015 16:35:50 -0800
Subject: [PATCH 104/732] Update index.apt

---
 src/site/apt/index.apt | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/site/apt/index.apt b/src/site/apt/index.apt
index 01347874fd16..98928898bbd3 100644
--- a/src/site/apt/index.apt
+++ b/src/site/apt/index.apt
@@ -10,14 +10,14 @@ GCloud Java: Idiomatic Java Client for Google Cloud Platform services.
  	
 * Links
   
-   	* {{{https://github.com/aozarov/git-demo}GitHub repository}}
+   	* {{{https://github.com/GoogleCloudPlatform/gcloud-java}GitHub repository}}
    	
-   	* {{{http://aozarov.github.io/git-demo/apidocs/index.html}Javadocs}}
+   	* {{{http://GoogleCloudPlatform.github.io/gcloud-java/apidocs/index.html}Javadocs}}
    	
-   	* {{{http://aozarov.github.io/git-demo/dependencies.html}Dependencies}}
+   	* {{{http://GoogleCloudPlatform.github.io/gcloud-java/dependencies.html}Dependencies}}
    	
-   	* {{{https://travis-ci.org/aozarov/git-demo} Continous Integration System (Travis-CI)}}
+   	* {{{https://travis-ci.org/GoogleCloudPlatform/gcloud-java} Continous Integration System (Travis-CI)}}
    	
-   	* {{{https://github.com/aozarov/git-demo/issues?page=1&state=open}Issues}}
+   	* {{{https://github.com/GoogleCloudPlatform/gcloud-java/issues?page=1&state=open}Issues}}
 
-   	* {{{https://coveralls.io/r/aozarov/git-demo?branch=master}Coverage}}
+   	* {{{https://coveralls.io/r/GoogleCloudPlatform/gcloud-java/issues?branch=master}Coverage}}

From 6a3354a5478aa31b98241325976b79af9a84ce7e Mon Sep 17 00:00:00 2001
From: Arie 
Date: Mon, 9 Feb 2015 16:53:11 -0800
Subject: [PATCH 105/732] Update index.apt

---
 src/site/apt/index.apt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/site/apt/index.apt b/src/site/apt/index.apt
index 98928898bbd3..0d4999bb7764 100644
--- a/src/site/apt/index.apt
+++ b/src/site/apt/index.apt
@@ -20,4 +20,4 @@ GCloud Java: Idiomatic Java Client for Google Cloud Platform services.
    	
    	* {{{https://github.com/GoogleCloudPlatform/gcloud-java/issues?page=1&state=open}Issues}}
 
-   	* {{{https://coveralls.io/r/GoogleCloudPlatform/gcloud-java/issues?branch=master}Coverage}}
+   	* {{{https://coveralls.io/r/GoogleCloudPlatform/gcloud-java/}Coverage}}

From fe95d030979b92207a052c815b1aee3ae24d6e6b Mon Sep 17 00:00:00 2001
From: Arie 
Date: Mon, 9 Feb 2015 16:54:09 -0800
Subject: [PATCH 106/732] Update site.xml

---
 src/site/site.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/site/site.xml b/src/site/site.xml
index afe6c29f6042..f47cc45a401f 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -1,5 +1,6 @@
 
+         xsi:schemaLocation="http://maven.apache.org/DECORATION/1.1.0 http://maven.apache.org/xsd/decoration-1.1.0.xsd"
+         name="GCloud Java">
   
     org.apache.maven.skins
     maven-fluido-skin

From 0d4412540e535b955b2879911808c84808c2874d Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 11 Feb 2015 09:04:40 -0800
Subject: [PATCH 107/732] Fixes #6

---
 src/main/java/com/google/gcloud/datastore/DateTime.java     | 2 +-
 src/test/java/com/google/gcloud/datastore/DateTimeTest.java | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/com/google/gcloud/datastore/DateTimeTest.java

diff --git a/src/main/java/com/google/gcloud/datastore/DateTime.java b/src/main/java/com/google/gcloud/datastore/DateTime.java
index 3b7e85a2e736..97df17ba840b 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTime.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTime.java
@@ -78,7 +78,7 @@ public Calendar toCalendar() {
   }
 
   public static DateTime now() {
-    return new DateTime(System.nanoTime() / 1000L);
+    return copyFrom(new Date());
   }
 
   public static DateTime copyFrom(Date date) {
diff --git a/src/test/java/com/google/gcloud/datastore/DateTimeTest.java b/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
new file mode 100644
index 000000000000..43d84970925a
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
@@ -0,0 +1 @@
+package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;

import org.junit.Test;

import java.util.Calendar;

public class DateTimeTest {

  @Test
  public void testTimestampMicroseconds() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis() * 1000, date.timestampMicroseconds());
  }

  @Test
  public void testTimestampMillis() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis(), date.timestampMillis());
  }

  @Test
  public void testToDate() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTime(), date.toDate());
  }

  @Test
  public void testToCalendar() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal, date.toCalendar());
  }

  @Test
  public void testNow() throws Exception {
    Calendar cal1 = Calendar.getInstance();
    DateTime now = DateTime.now();
    Calendar cal2 = Calendar.getInstance();
    assertTrue(now.timestampMillis() >= cal1.getTimeInMillis());
    assertTrue(now.timestampMillis() <= cal2.getTimeInMillis());
  }

  @Test
  public void testCopyFrom() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date1 = DateTime.copyFrom(cal);
    DateTime date2 = DateTime.copyFrom(cal.getTime());
    cal.add(Calendar.DATE, 1);
    DateTime date3 = DateTime.copyFrom(cal.getTime());
    assertEquals(date1, date2);
    assertNotEquals(date1, date3);
  }
}
\ No newline at end of file

From 705c1f04eb3c5d2cb27e421a374b693ac0955305 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 11 Feb 2015 18:44:40 -0800
Subject: [PATCH 108/732] adding more tests

---
 .../datastore/BaseDatastoreBatchWriter.java   | 23 ++++++++++++-------
 .../google/gcloud/datastore/BatchImpl.java    | 16 +++++++------
 .../com/google/gcloud/datastore/Cursor.java   | 12 +++++++---
 src/site/site.xml                             | 16 +++++++++++++
 .../BaseDatastoreBatchWriterTest.java         |  1 +
 .../gcloud/datastore/BaseEntityTest.java      |  1 +
 .../com/google/gcloud/datastore/BlobTest.java |  1 +
 .../google/gcloud/datastore/CursorTest.java   |  1 +
 .../google/gcloud/datastore/DateTimeTest.java |  2 +-
 9 files changed, 54 insertions(+), 19 deletions(-)
 create mode 100644 src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/BlobTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/CursorTest.java

diff --git a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
index f2f60ef5d30f..311965e54eac 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
@@ -1,22 +1,29 @@
 /*
  * Copyright 2015 Google Inc. All Rights Reserved.
  *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ *       http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
 
-import java.util.*;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Base class for DatastoreBatchWriter.
diff --git a/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
index 305bf4fbcf9f..0d7eb704d1eb 100644
--- a/src/main/java/com/google/gcloud/datastore/BatchImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -1,15 +1,17 @@
 /*
  * Copyright 2015 Google Inc. All Rights Reserved.
  *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ *       http://www.apache.org/licenses/LICENSE-2.0
  *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package com.google.gcloud.datastore;
diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/src/main/java/com/google/gcloud/datastore/Cursor.java
index 037e0ed7ed46..42a8cee8e5a2 100644
--- a/src/main/java/com/google/gcloud/datastore/Cursor.java
+++ b/src/main/java/com/google/gcloud/datastore/Cursor.java
@@ -23,8 +23,11 @@
 import com.google.api.services.datastore.DatastoreV1.Value;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.base.Preconditions;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.TextFormat;
+import com.google.protobuf.TextFormat.ParseException;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
@@ -41,6 +44,7 @@ public final class Cursor extends Serializable {
   private final transient ByteString byteString;
 
   Cursor(ByteString byteString) {
+    Preconditions.checkArgument(byteString.isValidUtf8(), "content is not a valid UTF-8");
     this.byteString = byteString;
   }
 
@@ -73,7 +77,7 @@ ByteString byteString() {
    */
   public String toUrlSafe() {
     try {
-      return URLEncoder.encode(toPb().toString(), UTF_8.name());
+      return URLEncoder.encode(TextFormat.printToString(toPb()), UTF_8.name());
     } catch (UnsupportedEncodingException e) {
       throw new IllegalStateException("Unexpected encoding exception", e);
     }
@@ -85,8 +89,10 @@ public String toUrlSafe() {
   public static Cursor fromUrlSafe(String urlSafe) {
     try {
       String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name());
-      return fromPb(DatastoreV1.Value.parseFrom(ByteString.copyFromUtf8(utf8Str)));
-    } catch (UnsupportedEncodingException | InvalidProtocolBufferException e) {
+      DatastoreV1.Value.Builder builder = DatastoreV1.Value.newBuilder();
+      TextFormat.merge(utf8Str, builder);
+      return fromPb(builder.build());
+    } catch (UnsupportedEncodingException | ParseException e) {
       throw new IllegalStateException("Unexpected decoding exception", e);
     }
   }
diff --git a/src/site/site.xml b/src/site/site.xml
index f47cc45a401f..db19e4a3877a 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -1,3 +1,19 @@
+
+
 
diff --git a/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
new file mode 100644
index 000000000000..8d440407a9e4
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;

import org.junit.Before;
import org.junit.Test;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder("dataset1", "kind1", 1).build();
  private static final Key KEY3 = Key.builder("dataset1", "kind1", 2).build();
  private static final PartialKey PARTIAL_KEY = PartialKey.builder("dataset1", "kind1").build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final PartialEntity PARTIAL_ENTITY_1 = Entity.builder(PARTIAL_KEY).build();
  private static final PartialEntity PARTIAL_ENTITY_2 =
      PartialEntity.builder(PARTIAL_KEY).set("name", "dan").build();


  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    protected DatastoreBatchWriter() {
      super("test");
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @Test
  public void testAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(ENTITY2.toPb())
        .addInsert(ENTITY3.toPb())
        .build();
    batchWriter.add(ENTITY1, ENTITY2);
    batchWriter.add(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddAutoId() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(PARTIAL_ENTITY_1.toPb())
        .addInsertAutoId(PARTIAL_ENTITY_2.toPb())
        .build();
    batchWriter.add(ENTITY1, PARTIAL_ENTITY_1);
    batchWriter.add(PARTIAL_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAutoIdWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(PARTIAL_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(PARTIAL_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.add(PARTIAL_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
new file mode 100644
index 000000000000..f3f7a5f62b09
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.protobuf.InvalidProtocolBufferException;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final PartialKey PARTIAL_KEY = PartialKey.builder("ds1", "k1").build();
  private static final PartialEntity PARTIAL_ENTITY = PartialEntity.builder(PARTIAL_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override
    public BaseEntity build() {

      return new BaseEntity(ImmutableSortedMap.copyOf(properties)) {
        @Override
        protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {

        }

        @Override
        protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
          return null;
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/BlobTest.java b/src/test/java/com/google/gcloud/datastore/BlobTest.java
new file mode 100644
index 000000000000..e78023101e48
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/BlobTest.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;

import org.junit.Before;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Random;

public class BlobTest {

  private byte[] bytes1 = new byte[10];
  private byte[] bytes2 = new byte[11];
  private Blob blob1;
  private Blob blob2;


  @Before
  public void setUp() {
    Random rnd = new Random();
    rnd.nextBytes(bytes1);
    rnd.nextBytes(bytes2);
    blob1 = Blob.copyFrom(bytes1);
    blob2 = Blob.copyFrom(bytes2);
  }

  @Test
  public void testEquals() throws Exception {
    assertEquals(blob1, blob1);
    assertEquals(blob1, Blob.copyFrom(bytes1));
    assertNotEquals(blob1, blob2);
  }

  @Test
  public void testLength() throws Exception {
    assertEquals(bytes1.length, blob1.length());
    assertEquals(bytes2.length, blob2.length());
  }

  @Test
  public void testToByteArray() throws Exception {
    assertArrayEquals(bytes1, blob1.toByteArray());
    assertArrayEquals(bytes2, blob2.toByteArray());
  }

  @Test
  public void testAsReadOnlyByteBuffer() throws Exception {
    ByteBuffer buffer = blob1.asReadOnlyByteBuffer();
    byte[] bytes = new byte[bytes1.length];
    buffer.get(bytes);
    assertFalse(buffer.hasRemaining());
    assertArrayEquals(bytes1, bytes);
  }

  @Test
  public void testAsInputStream() throws Exception {
    byte[] bytes = new byte[bytes1.length];
    InputStream in = blob1.asInputStream();
    assertEquals(bytes1.length, in.read(bytes));
    assertEquals(-1, in.read());
    assertArrayEquals(bytes1, bytes);
  }

  @Test
  public void testCopyTo() throws Exception {
    byte[] bytes = new byte[bytes1.length];
    blob1.copyTo(bytes);
    assertArrayEquals(bytes1, bytes);

    ByteBuffer buffer = ByteBuffer.allocate(bytes1.length);
    blob1.copyTo(buffer);
    buffer.flip();
    bytes = new byte[bytes1.length];
    buffer.get(bytes);
    assertFalse(buffer.hasRemaining());
    assertArrayEquals(bytes1, bytes);
  }

  @Test
  public void testCopyFrom() throws Exception {
    Blob blob = Blob.copyFrom(ByteBuffer.wrap(bytes1));
    assertEquals(blob1, blob);
    assertArrayEquals(bytes1, blob.toByteArray());

    blob = Blob.copyFrom(new ByteArrayInputStream(bytes2));
    assertEquals(blob2, blob);
    assertArrayEquals(bytes2, blob.toByteArray());
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/CursorTest.java b/src/test/java/com/google/gcloud/datastore/CursorTest.java
new file mode 100644
index 000000000000..6806fd698331
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/CursorTest.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;

import com.google.protobuf.ByteString;

import org.junit.Before;
import org.junit.Test;

public class CursorTest {

  private byte[] bytes1 = {1, 2, 3, '%', '<', '+'};
  private byte[] bytes2 = {10, 20, 30};
  private Cursor cursor1;
  private Cursor cursor2;

  @Before
  public void setUp() throws Exception {
    cursor1 = new Cursor(ByteString.copyFrom(bytes1));
    cursor2 = new Cursor(ByteString.copyFrom(bytes2));
  }

  @Test
  public void testToFromUrlSafe() throws Exception {
    String urlSafe = cursor1.toUrlSafe();
    assertEquals(cursor1, Cursor.fromUrlSafe(urlSafe));
  }

  @Test
  public void testCopyFrom() throws Exception {
    Cursor cursor = Cursor.copyFrom(bytes2);
    assertEquals(cursor2, cursor);
    assertNotEquals(cursor1, cursor);
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DateTimeTest.java b/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
index 43d84970925a..a7131e04e89c 100644
--- a/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
@@ -1 +1 @@
-package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;

import org.junit.Test;

import java.util.Calendar;

public class DateTimeTest {

  @Test
  public void testTimestampMicroseconds() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis() * 1000, date.timestampMicroseconds());
  }

  @Test
  public void testTimestampMillis() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis(), date.timestampMillis());
  }

  @Test
  public void testToDate() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTime(), date.toDate());
  }

  @Test
  public void testToCalendar() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal, date.toCalendar());
  }

  @Test
  public void testNow() throws Exception {
    Calendar cal1 = Calendar.getInstance();
    DateTime now = DateTime.now();
    Calendar cal2 = Calendar.getInstance();
    assertTrue(now.timestampMillis() >= cal1.getTimeInMillis());
    assertTrue(now.timestampMillis() <= cal2.getTimeInMillis());
  }

  @Test
  public void testCopyFrom() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date1 = DateTime.copyFrom(cal);
    DateTime date2 = DateTime.copyFrom(cal.getTime());
    cal.add(Calendar.DATE, 1);
    DateTime date3 = DateTime.copyFrom(cal.getTime());
    assertEquals(date1, date2);
    assertNotEquals(date1, date3);
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;

import org.junit.Test;

import java.util.Calendar;

public class DateTimeTest {

  @Test
  public void testTimestampMicroseconds() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis() * 1000, date.timestampMicroseconds());
  }

  @Test
  public void testTimestampMillis() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis(), date.timestampMillis());
  }

  @Test
  public void testToDate() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTime(), date.toDate());
  }

  @Test
  public void testToCalendar() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal, date.toCalendar());
  }

  @Test
  public void testNow() throws Exception {
    Calendar cal1 = Calendar.getInstance();
    DateTime now = DateTime.now();
    Calendar cal2 = Calendar.getInstance();
    assertTrue(now.timestampMillis() >= cal1.getTimeInMillis());
    assertTrue(now.timestampMillis() <= cal2.getTimeInMillis());
  }

  @Test
  public void testCopyFrom() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date1 = DateTime.copyFrom(cal);
    DateTime date2 = DateTime.copyFrom(cal.getTime());
    cal.add(Calendar.DATE, 1);
    DateTime date3 = DateTime.copyFrom(cal.getTime());
    assertEquals(date1, date2);
    assertNotEquals(date1, date3);
  }
}

\ No newline at end of file

From ac75af9d9e7bb1e3523252d3678ae9d18df70198 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Fri, 13 Feb 2015 14:29:25 -0800
Subject: [PATCH 109/732] Update README.md

Fix apidocs link
---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7bbb1da1f68d..3f8dcf9182bf 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,8 @@ Java idiomatic client for Google Cloud Platform services. Supported APIs include
 > Note: This package is a work-in-progress, and may occasionally
 > make backwards-incompatible changes.
 
-Documentation and examples are available [here](https://googlecloudeplatform.github.com/gcloud-java/apidocs).
+
+Documentation and examples are available [here](http://googlecloudplatform.github.io/gcloud-java/apidocs).
 
 ## Google Cloud Datastore
 

From 09779e91e6b9fdc16d2c7aab289a0da3c47bf9e6 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 13 Feb 2015 17:37:46 -0800
Subject: [PATCH 110/732] more tests

---
 .../com/google/gcloud/datastore/BaseKey.java  |   1 +
 .../java/com/google/gcloud/datastore/Key.java |  11 +-
 .../google/gcloud/datastore/BaseKeyTest.java  | 125 +++++++++++++++
 .../google/gcloud/datastore/EntityTest.java   |  56 +++++++
 .../com/google/gcloud/datastore/KeyTest.java  |  79 ++++++++++
 .../gcloud/datastore/PartialEntityTest.java   |  62 ++++++++
 .../gcloud/datastore/PartialKeyTest.java      |  45 ++++++
 .../gcloud/datastore/PathElementTest.java     |  73 +++++++++
 .../google/gcloud/datastore/ValueTest.java    | 149 ++++++++++++++++++
 9 files changed, 596 insertions(+), 5 deletions(-)
 create mode 100644 src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/EntityTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/KeyTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/PartialKeyTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/PathElementTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/ValueTest.java

diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index 90736614b5e1..3a1ff342e4e3 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -107,6 +107,7 @@ public B namespace(String namespace) {
   }
 
   BaseKey(String dataset, String namespace, ImmutableList path) {
+    Preconditions.checkArgument(!path.isEmpty(), "Path must not be empty");
     this.dataset = dataset;
     this.namespace = namespace;
     this.path = path;
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index 97559478293c..1555b109004c 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -21,8 +21,8 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
-import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.TextFormat;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
@@ -138,7 +138,7 @@ public Object nameOrId() {
    */
   public String toUrlSafe() {
     try {
-      return URLEncoder.encode(toString(), UTF_8.name());
+      return URLEncoder.encode(TextFormat.printToString(toPb()), UTF_8.name());
     } catch (UnsupportedEncodingException e) {
       throw new IllegalStateException("Unexpected encoding exception", e);
     }
@@ -152,11 +152,12 @@ public String toUrlSafe() {
   public static Key fromUrlSafe(String urlSafe) {
     try {
       String utf8Str = URLDecoder.decode(urlSafe, UTF_8.name());
-      DatastoreV1.Key keyPb = DatastoreV1.Key.parseFrom(ByteString.copyFromUtf8(utf8Str));
-      return fromPb(keyPb);
+      DatastoreV1.Key.Builder builder = DatastoreV1.Key.newBuilder();
+      TextFormat.merge(utf8Str, builder);
+      return fromPb(builder.build());
     } catch (UnsupportedEncodingException e) {
       throw new IllegalStateException("Unexpected decoding exception", e);
-    } catch (InvalidProtocolBufferException e) {
+    } catch (TextFormat.ParseException e) {
       throw new IllegalArgumentException("Could not parse key", e);
     }
   }
diff --git a/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java b/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
new file mode 100644
index 000000000000..850a71a127bd
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class BaseKeyTest {
+
+  private class Builder extends BaseKey.Builder {
+
+    Builder(String dataset) {
+      super(dataset);
+    }
+
+    Builder(String dataset, String kind) {
+      super(dataset, kind);
+    }
+
+    @Override
+    protected BaseKey build() {
+      ImmutableList.Builder path = ImmutableList.builder();
+      path.addAll(ancestors);
+      path.add(PathElement.of(kind));
+      return new BaseKey(dataset, namespace, path.build()) {
+        @Override
+        protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+          return null;
+        }
+      };
+    }
+  }
+
+  @Test
+  public void testDataset() throws Exception {
+    Builder builder = new Builder("ds1", "k");
+    BaseKey key = builder.build();
+    assertEquals("ds1", key.dataset());
+    key = builder.dataset("ds2").build();
+    assertEquals("ds2", key.dataset());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testBadDatasetInConstructor() throws Exception {
+    new Builder(" ", "k");
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testBadDatasetInSetter() throws Exception {
+    Builder builder = new Builder("d", "k");
+    builder.dataset(" ");
+  }
+
+  @Test
+  public void testNamespace() throws Exception {
+    Builder builder = new Builder("ds", "k");
+    BaseKey key = builder.build();
+    assertNull(key.namespace());
+    key = builder.namespace("ns").build();
+    assertEquals("ns", key.namespace());
+  }
+
+  @Test
+  public void testKind() throws Exception {
+    Builder builder = new Builder("ds", "k1");
+    BaseKey key = builder.build();
+    assertEquals("k1", key.kind());
+    key = builder.kind("k2").build();
+    assertEquals("k2", key.kind());
+  }
+
+  @Test(expected = NullPointerException.class)
+  public void testNoKind() throws Exception {
+    Builder builder = new Builder("ds");
+    builder.build();
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testBadKindInConstructor() throws Exception {
+    new Builder("ds", "");
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testBadKindInSetter() throws Exception {
+    Builder builder = new Builder("ds", "k1");
+    builder.kind("");
+  }
+
+  @Test
+  public void testAncestors() throws Exception {
+    Builder builder = new Builder("ds", "k");
+    BaseKey key = builder.build();
+    assertTrue(key.ancestors().isEmpty());
+    List path = new ArrayList<>();
+    path.add(PathElement.of("p1","v1"));
+    key = builder.ancestors(path.get(0)).build();
+    assertEquals(path, key.ancestors());
+    path.add(PathElement.of("p2","v2"));
+    key = builder.ancestors(path.get(1)).build();
+    assertEquals(path, key.ancestors());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/EntityTest.java b/src/test/java/com/google/gcloud/datastore/EntityTest.java
new file mode 100644
index 000000000000..a60ad3b18b80
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/EntityTest.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import org.junit.Test;
+
+public class EntityTest {
+
+  private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
+  private static final Key KEY2 = Key.builder("ds1", "k2", 1).build();
+  private static final PartialKey PARTIAL_KEY = PartialKey.builder("ds1", "k2").build();
+  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
+  private static final PartialEntity PARTIAL_ENTITY =
+      PartialEntity.builder(PARTIAL_KEY).set("a", "b").build();
+
+  @Test
+  public void testGetKey() throws Exception {
+    assertEquals(KEY1, ENTITY.key());
+    assertEquals("bar", ENTITY.getString("foo"));
+  }
+
+  @Test
+  public void testCopyFrom() throws Exception {
+    Entity.Builder builder = Entity.builder(ENTITY);
+    assertEquals(ENTITY, builder.build());
+    Entity entity = builder.key(KEY2).build();
+    assertNotEquals(ENTITY, entity);
+    assertEquals(KEY2, entity.key());
+    assertEquals(ENTITY.properties(), entity.properties());
+  }
+
+  @Test
+  public void testCopyFromPartialEntity() throws Exception {
+    Entity.Builder builder = Entity.builder(KEY2, PARTIAL_ENTITY);
+    Entity entity = builder.build();
+    assertNotEquals(PARTIAL_ENTITY, entity);
+    assertEquals(PARTIAL_ENTITY.properties(), entity.properties());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/KeyTest.java b/src/test/java/com/google/gcloud/datastore/KeyTest.java
new file mode 100644
index 000000000000..1fdcc5394e7e
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/KeyTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class KeyTest {
+
+  @Test
+  public void testHasId() throws Exception {
+    Key.Builder builder = Key.builder("d", "k", 10);
+    Key key = builder.build();
+    assertTrue(key.hasId());
+    key = builder.name("bla").build();
+    assertFalse(key.hasId());
+  }
+
+  @Test
+  public void testId() throws Exception {
+    Key.Builder builder = Key.builder("d", "k", 10);
+    Key key = builder.build();
+    assertEquals(Long.valueOf(10), key.id());
+    key = builder.id(100).build();
+    assertEquals(Long.valueOf(100), key.id());
+  }
+
+  @Test
+  public void testHasName() throws Exception {
+    Key.Builder builder = Key.builder("d", "k", "n");
+    Key key = builder.build();
+    assertTrue(key.hasName());
+    key = builder.id(1).build();
+    assertFalse(key.hasName());
+  }
+
+  @Test
+  public void testName() throws Exception {
+    Key.Builder builder = Key.builder("d", "k", "n");
+    Key key = builder.build();
+    assertEquals("n", key.name());
+    key = builder.name("o").build();
+    assertEquals("o", key.name());
+  }
+
+  @Test
+  public void testNameOrId() throws Exception {
+    Key.Builder builder = Key.builder("d", "k", "n");
+    Key key = builder.build();
+    assertEquals("n", key.nameOrId());
+    key = builder.id(1).build();
+    assertEquals(Long.valueOf(1), key.nameOrId());
+  }
+
+  @Test
+  public void testToAndFromUrlSafe() throws Exception {
+    Key key = Key.builder("d", "k", "n").build();
+    String urlSafe = key.toUrlSafe();
+    Key copy = Key.fromUrlSafe(urlSafe);
+    assertEquals(key, copy);
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java b/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
new file mode 100644
index 000000000000..1e5f677c024f
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Test;
+
+public class PartialEntityTest {
+
+  private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
+  private static final PartialKey PARTIAL_KEY = PartialKey.builder("ds1", "k2").build();
+  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
+  private static final PartialEntity PARTIAL_ENTITY =
+      PartialEntity.builder(PARTIAL_KEY).set("a", "b").build();
+
+  @Test
+  public void testGetKey() throws Exception {
+    assertEquals(PARTIAL_KEY, PARTIAL_ENTITY.key());
+    assertEquals("b", PARTIAL_ENTITY.getString("a"));
+  }
+
+  @Test
+  public void testNoKey() throws Exception {
+    PartialEntity entity = PartialEntity.builder().set("foo", "bar").build();
+    assertNull(entity.key());
+    assertEquals("bar", entity.getString("foo"));
+  }
+
+  @Test
+  public void testCopyFromPartialEntity() throws Exception {
+    PartialEntity entity = Entity.builder(PARTIAL_ENTITY).build();
+    assertEquals(PARTIAL_ENTITY, entity);
+    entity = Entity.builder(PARTIAL_ENTITY).key(KEY1).build();
+    assertNotEquals(PARTIAL_ENTITY, entity);
+    assertEquals(PARTIAL_ENTITY.properties(), entity.properties());
+    assertNotEquals(PARTIAL_ENTITY.key(), entity.key());
+    assertEquals(KEY1, entity.key());
+  }
+
+  @Test
+  public void testCopyFromEntity() throws Exception {
+    PartialEntity entity = Entity.builder(ENTITY).build();
+    assertEquals(ENTITY, entity);
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/PartialKeyTest.java b/src/test/java/com/google/gcloud/datastore/PartialKeyTest.java
new file mode 100644
index 000000000000..36cf395ec842
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/PartialKeyTest.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class PartialKeyTest {
+
+  @Test
+  public void testBuilders() throws Exception {
+    PartialKey pk1 = PartialKey.builder("ds", "kind1").build();
+    assertEquals("ds", pk1.dataset());
+    assertEquals("kind1", pk1.kind());
+    assertTrue(pk1.ancestors().isEmpty());
+
+    Key parent = Key.builder("ds", "kind2", 10).build();
+    PartialKey pk2 = PartialKey.builder(parent, "kind3").build();
+    assertEquals("ds", pk2.dataset());
+    assertEquals("kind3", pk2.kind());
+    assertEquals(parent.path(), pk2.ancestors());
+
+    assertEquals(pk2, PartialKey.builder(pk2).build());
+    PartialKey pk3 = PartialKey.builder(pk2).kind("kind4").build();
+    assertEquals("ds", pk3.dataset());
+    assertEquals("kind4", pk3.kind());
+    assertEquals(parent.path(), pk3.ancestors());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/PathElementTest.java b/src/test/java/com/google/gcloud/datastore/PathElementTest.java
new file mode 100644
index 000000000000..393521ff08b9
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/PathElementTest.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class PathElementTest {
+
+  private static final PathElement PE_1 = PathElement.of("k1");
+  private static final PathElement PE_2 = PathElement.of("k2", "n");
+  private static final PathElement PE_3 = PathElement.of("k3", 1);
+
+  @Test
+  public void testKind() throws Exception {
+    assertEquals("k1", PE_1.kind());
+    assertEquals("k2", PE_2.kind());
+    assertEquals("k3", PE_3.kind());
+  }
+
+  @Test
+  public void testHasId() throws Exception {
+    assertFalse(PE_1.hasId());
+    assertFalse(PE_2.hasId());
+    assertTrue(PE_3.hasId());
+  }
+
+  @Test
+  public void testId() throws Exception {
+    assertNull(PE_1.id());
+    assertNull(PE_2.id());
+    assertEquals(Long.valueOf(1), PE_3.id());
+  }
+
+  @Test
+  public void testHasName() throws Exception {
+    assertFalse(PE_1.hasName());
+    assertTrue(PE_2.hasName());
+    assertFalse(PE_3.hasName());
+  }
+
+  @Test
+  public void testName() throws Exception {
+    assertNull(PE_1.name());
+    assertEquals("n", PE_2.name());
+    assertNull(PE_3.name());
+  }
+
+  @Test
+  public void testNameOrId() throws Exception {
+    assertNull(PE_1.nameOrId());
+    assertEquals("n", PE_2.nameOrId());
+    assertEquals(Long.valueOf(1), PE_3.nameOrId());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/ValueTest.java b/src/test/java/com/google/gcloud/datastore/ValueTest.java
new file mode 100644
index 000000000000..212f3ba32afb
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.primitives.Primitives;
+import com.google.gcloud.datastore.Value.Type;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Method;
+import java.util.Map;
+
+public class ValueTest {
+
+  private static final Key KEY = Key.builder("ds", "kind", 1).build();
+  private static final Blob BLOB = Blob.copyFrom(new byte[]{});
+  private static final DateTime DATE_TIME = DateTime.now();
+  private static final Entity ENTITY = Entity.builder(KEY).set("FOO", "BAR").build();
+  private static final NullValue NULL_VALUE = NullValue.of();
+  private static final StringValue STRING_VALUE = StringValue.of("hello");
+  private static final RawValue RAW_VALUE = RawValue.of(STRING_VALUE.toPb());
+  private static final ImmutableMap TYPES = ImmutableMap.builder()
+      .put(Type.NULL, new Object[] {NullValue.class, NULL_VALUE.get()})
+      .put(Type.KEY, new Object[] {KeyValue.class, KEY})
+      .put(Type.BLOB, new Object[] {BlobValue.class, BLOB})
+      .put(Type.BOOLEAN, new Object[] {BooleanValue.class, Boolean.TRUE})
+      .put(Type.DATE_TIME, new Object[] {DateTimeValue.class, DATE_TIME})
+      .put(Type.DOUBLE, new Object[] {DoubleValue.class, 1.25D})
+      .put(Type.ENTITY, new Object[] {EntityValue.class, ENTITY})
+      .put(Type.LIST,
+          new Object[] {ListValue.class, ImmutableList.of(NULL_VALUE, STRING_VALUE, RAW_VALUE)})
+      .put(Type.LONG, new Object[] {LongValue.class, 123L})
+      .put(Type.RAW_VALUE, new Object[] {RawValue.class, RAW_VALUE.get()})
+      .put(Type.STRING, new Object[] {StringValue.class, STRING_VALUE.get()})
+      .build();
+
+  private ImmutableMap> typeToValue;
+
+  @Before
+  public void setUp() throws Exception {
+    ImmutableMap.Builder> builder = ImmutableMap.builder();
+    for (Type type : Type.values()) {
+      Object[] values = TYPES.get(type);
+      Class> valueClass = (Class>) values[0];
+      Object value = values[1];
+      if (value == null) {
+        Method method = valueClass.getMethod("of");
+        builder.put(type, (Value) method.invoke(null));
+      } else {
+        boolean found = false;
+        for (Method method : valueClass.getDeclaredMethods()) {
+          if (method.getName().equals("of")) {
+            Class paramType = method.getParameterTypes()[0];
+            if (paramType.isPrimitive()) {
+              paramType = Primitives.wrap(paramType);
+            }
+            if (paramType.isAssignableFrom(value.getClass())) {
+              builder.put(type, (Value) method.invoke(null, value));
+              found = true;
+              break;
+            }
+          }
+        }
+        assertTrue("Could not find an of method for " + valueClass, found);
+      }
+    }
+    typeToValue = builder.build();
+  }
+
+  @Test
+  public void testType() throws Exception {
+    for (Map.Entry> entry : typeToValue.entrySet()) {
+      assertEquals(entry.getKey(), entry.getValue().type());
+    }
+  }
+
+  @Test
+  public void testHasIndexed() throws Exception {
+    for (Map.Entry> entry : typeToValue.entrySet()) {
+      Type type = entry.getKey();
+      Boolean indexed = entry.getValue().hasIndexed();
+      switch (type) {
+        case ENTITY:
+          assertTrue(indexed);
+          break;
+        default:
+          assertFalse(indexed);
+          break;
+      }
+    }
+  }
+
+  @Test
+  public void testIndexed() throws Exception {
+    for (Map.Entry> entry : typeToValue.entrySet()) {
+      Type type = entry.getKey();
+      Boolean indexed = entry.getValue().indexed();
+      switch (type) {
+        case ENTITY:
+          assertFalse(indexed);
+          break;
+        default:
+          assertNull(indexed);
+          break;
+      }
+    }
+  }
+
+  @Test
+  public void testHasMeaning() throws Exception {
+
+  }
+
+  @Test
+  public void testMeaning() throws Exception {
+
+  }
+
+  @Test
+  public void testGet() throws Exception {
+
+  }
+
+  @Test
+  public void testToBuilder() throws Exception {
+
+  }
+}

From b5d5cf00cd54ba7714a2363875750e3f5222d42d Mon Sep 17 00:00:00 2001
From: Arie 
Date: Tue, 17 Feb 2015 09:26:49 -0800
Subject: [PATCH 111/732] Create LICENSE

---
 LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 201 insertions(+)
 create mode 100644 LICENSE

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 000000000000..4eedc0116add
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

From 40e5ae0f379d2ee4a359e877690d1b13fdf320a8 Mon Sep 17 00:00:00 2001
From: Misha Brukman 
Date: Tue, 17 Feb 2015 13:32:28 -0500
Subject: [PATCH 112/732] Syntax highlight Java code snippet.

---
 README.md | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index 3f8dcf9182bf..2ffe0742ccc7 100644
--- a/README.md
+++ b/README.md
@@ -26,26 +26,28 @@ consistency for all other queries.
 Follow the [activation instructions][cloud-datastore-activation] to use the Google
 Cloud Datastore API with your project.
 
-    import com.google.gcloud.datastore.DatastoreService;
-    import com.google.gcloud.datastore.DatastoreServiceFactory;
-    import com.google.gcloud.datastore.DatastoreServiceOptions;
-    import com.google.gcloud.datastore.Entity;
-    import com.google.gcloud.datastore.Key;
-    import com.google.gcloud.datastore.KeyFactory;
-
-    DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset("...").build();
-    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
-    KeyFactory keyFactory = new KeyFactory(datastore).kind("...");
-    Key key = keyFactory.newKey(keyName);
-    Entity entity = datastore.get(key);
-    if (entity == null) {
-      entity = Entity.builder(key)
-          .set("name", "John Do")
-          .set("age", 30)
-          .set("updated", false)
-          .build();
-      datastore.put(entity);
-    }
+```java
+import com.google.gcloud.datastore.DatastoreService;
+import com.google.gcloud.datastore.DatastoreServiceFactory;
+import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.Key;
+import com.google.gcloud.datastore.KeyFactory;
+
+DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset("...").build();
+DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
+KeyFactory keyFactory = new KeyFactory(datastore).kind("...");
+Key key = keyFactory.newKey(keyName);
+Entity entity = datastore.get(key);
+if (entity == null) {
+  entity = Entity.builder(key)
+      .set("name", "John Do")
+      .set("age", 30)
+      .set("updated", false)
+      .build();
+  datastore.put(entity);
+}
+```
 
 ## Contributing
 

From ffa1b5b568e7048ffdbeb3a906208bff235e54f8 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 17 Feb 2015 16:46:52 -0800
Subject: [PATCH 113/732] Value tests

---
 .../gcloud/datastore/BlobValueTest.java       | 53 +++++++++++++++++
 .../gcloud/datastore/BooleanValueTest.java    | 51 ++++++++++++++++
 .../gcloud/datastore/DateTimeValueTest.java   | 53 +++++++++++++++++
 .../gcloud/datastore/DoubleValueTest.java     | 53 +++++++++++++++++
 .../gcloud/datastore/EntityValueTest.java     | 55 +++++++++++++++++
 .../google/gcloud/datastore/KeyValueTest.java | 53 +++++++++++++++++
 .../gcloud/datastore/LongValueTest.java       | 53 +++++++++++++++++
 .../gcloud/datastore/NullValueTest.java       | 52 ++++++++++++++++
 .../gcloud/datastore/StringValueTest.java     | 53 +++++++++++++++++
 .../google/gcloud/datastore/ValueTest.java    | 59 ++++++++++++++++++-
 10 files changed, 534 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/com/google/gcloud/datastore/BlobValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/EntityValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/KeyValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/LongValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/NullValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/StringValueTest.java

diff --git a/src/test/java/com/google/gcloud/datastore/BlobValueTest.java b/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
new file mode 100644
index 000000000000..08709b846048
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class BlobValueTest {
+
+  private static final Blob CONTENT = Blob.copyFrom(new byte[] {1, 2});
+
+  @Test
+  public void testToBuilder() throws Exception {
+    BlobValue value = BlobValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    BlobValue value = BlobValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    BlobValue.Builder builder = BlobValue.builder(CONTENT);
+    BlobValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java b/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
new file mode 100644
index 000000000000..178ed1707152
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class BooleanValueTest {
+
+  @Test
+  public void testToBuilder() throws Exception {
+    BooleanValue value = BooleanValue.of(true);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    BooleanValue value = BooleanValue.of(false);
+    assertFalse(value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    BooleanValue.Builder builder = BooleanValue.builder(true);
+    BooleanValue value = builder.meaning(1).indexed(true).build();
+    assertTrue(value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertTrue(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java b/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
new file mode 100644
index 000000000000..2c5436a3e25b
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class DateTimeValueTest {
+
+  private static final DateTime CONTENT = DateTime.now();
+
+  @Test
+  public void testToBuilder() throws Exception {
+    DateTimeValue value = DateTimeValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    DateTimeValue value = DateTimeValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    DateTimeValue.Builder builder = DateTimeValue.builder(CONTENT);
+    DateTimeValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java b/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
new file mode 100644
index 000000000000..c264c5f20586
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class DoubleValueTest {
+
+  private static final Double CONTENT = 1.25;
+
+  @Test
+  public void testToBuilder() throws Exception {
+    DoubleValue value = DoubleValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    DoubleValue value = DoubleValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    DoubleValue.Builder builder = DoubleValue.builder(CONTENT);
+    DoubleValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
new file mode 100644
index 000000000000..639c471a5155
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class EntityValueTest {
+
+  private static final Key KEY = Key.builder("ds", "kind", 1).build();
+  private static final Entity CONTENT = Entity.builder(KEY).set("FOO", "BAR").build();
+
+  @Test
+  public void testToBuilder() throws Exception {
+    EntityValue value = EntityValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    EntityValue value = EntityValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    EntityValue.Builder builder = EntityValue.builder(CONTENT);
+    EntityValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/KeyValueTest.java b/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
new file mode 100644
index 000000000000..ebbbf7101543
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class KeyValueTest {
+
+  private static final Key CONTENT = Key.builder("ds", "kind", 1).build();
+
+  @Test
+  public void testToBuilder() throws Exception {
+    KeyValue value = KeyValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    KeyValue value = KeyValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    KeyValue.Builder builder = KeyValue.builder(CONTENT);
+    KeyValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/LongValueTest.java b/src/test/java/com/google/gcloud/datastore/LongValueTest.java
new file mode 100644
index 000000000000..b0881ba7dd7a
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/LongValueTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class LongValueTest {
+
+  private static final Long CONTENT = 125L;
+
+  @Test
+  public void testToBuilder() throws Exception {
+    LongValue value = LongValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    LongValue value = LongValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    LongValue.Builder builder = LongValue.builder(CONTENT);
+    LongValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/NullValueTest.java b/src/test/java/com/google/gcloud/datastore/NullValueTest.java
new file mode 100644
index 000000000000..24fc7ff2e0b1
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/NullValueTest.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class NullValueTest {
+
+  @Test
+  public void testToBuilder() throws Exception {
+    NullValue value = NullValue.of();
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    NullValue value = NullValue.of();
+    assertNull(value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    NullValue.Builder builder = NullValue.builder();
+    NullValue value = builder.meaning(1).indexed(false).build();
+    assertNull(value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/StringValueTest.java b/src/test/java/com/google/gcloud/datastore/StringValueTest.java
new file mode 100644
index 000000000000..42e83b6a4850
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/StringValueTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class StringValueTest {
+
+  private static final String CONTENT = "hello world";
+
+  @Test
+  public void testToBuilder() throws Exception {
+    StringValue value = StringValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    StringValue value = StringValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    StringValue.Builder builder = StringValue.builder(CONTENT);
+    StringValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/ValueTest.java b/src/test/java/com/google/gcloud/datastore/ValueTest.java
index 212f3ba32afb..04364283ecac 100644
--- a/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -29,7 +29,9 @@
 import org.junit.Test;
 
 import java.lang.reflect.Method;
+import java.util.Collections;
 import java.util.Map;
+import java.util.Set;
 
 public class ValueTest {
 
@@ -57,6 +59,23 @@ public class ValueTest {
 
   private ImmutableMap> typeToValue;
 
+  private class TestBuilder extends Value.BaseBuilder, TestBuilder> {
+    TestBuilder() {
+      super(Type.LIST);
+    }
+
+    @Override
+    public Value build() {
+      return new Value(this) {
+
+        @Override
+        public TestBuilder toBuilder() {
+          return new TestBuilder().mergeFrom(this);
+        }
+      };
+    }
+  }
+
   @Before
   public void setUp() throws Exception {
     ImmutableMap.Builder> builder = ImmutableMap.builder();
@@ -109,6 +128,11 @@ public void testHasIndexed() throws Exception {
           break;
       }
     }
+
+    TestBuilder builder = new TestBuilder();
+    assertFalse(builder.build().hasIndexed());
+    assertTrue(builder.indexed(false).build().hasIndexed());
+    assertTrue(builder.indexed(true).build().hasIndexed());
   }
 
   @Test
@@ -125,25 +149,58 @@ public void testIndexed() throws Exception {
           break;
       }
     }
+
+    TestBuilder builder = new TestBuilder();
+    assertNull(builder.build().indexed());
+    assertFalse(builder.indexed(false).build().indexed());
+    assertTrue(builder.indexed(true).build().indexed());
   }
 
   @Test
   public void testHasMeaning() throws Exception {
+    for (Value value: typeToValue.values()) {
+      assertFalse(value.hasMeaning());
+    }
 
+    TestBuilder builder = new TestBuilder();
+    assertTrue(builder.meaning(10).build().hasMeaning());
   }
 
   @Test
   public void testMeaning() throws Exception {
+    for (Value value: typeToValue.values()) {
+      assertNull(value.meaning());
+    }
 
+    TestBuilder builder = new TestBuilder();
+    assertEquals(Integer.valueOf(10), builder.meaning(10).build().meaning());
   }
 
   @Test
   public void testGet() throws Exception {
+    for (Map.Entry> entry : typeToValue.entrySet()) {
+      Type type = entry.getKey();
+      Value value = entry.getValue();
+      assertEquals(TYPES.get(type)[1], value.get());
+    }
 
+    TestBuilder builder = new TestBuilder();
+    Set value = Collections.singleton("bla");
+    assertEquals(value, builder.set(value).build().get());
   }
 
   @Test
   public void testToBuilder() throws Exception {
-
+    Set content = Collections.singleton("bla");
+    Value.Builder builder = new TestBuilder();
+    builder.meaning(1).set(content).indexed(true);
+    Value value = builder.build();
+    builder = value.toBuilder();
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertTrue(value.indexed());
+    assertEquals(Type.LIST, value.type());
+    assertEquals(content, value.get());
+    assertEquals(value, builder.build());
   }
 }

From 2f3be49f3a028fe92a258ab547e2fa6f8b5f1160 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 18 Feb 2015 14:41:24 -0800
Subject: [PATCH 114/732] more tests

---
 .../datastore/DatastoreServiceOptions.java    | 58 +++++++-----
 .../google/gcloud/datastore/KeyFactory.java   |  4 +
 .../gcloud/datastore/ProjectionEntity.java    |  2 +-
 .../DatastoreServiceOptionsTest.java          | 67 ++++++++++++++
 .../gcloud/datastore/EntityValueTest.java     |  5 ++
 .../gcloud/datastore/KeyFactoryTest.java      | 90 +++++++++++++++++++
 .../gcloud/datastore/ListValueTest.java       | 66 ++++++++++++++
 .../datastore/ProjectionEntityTest.java       | 48 ++++++++++
 .../google/gcloud/datastore/RawValueTest.java | 54 +++++++++++
 9 files changed, 372 insertions(+), 22 deletions(-)
 create mode 100644 src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/ListValueTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/RawValueTest.java

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 56acdb340c28..f3d90ab38597 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -46,6 +46,7 @@ public class DatastoreServiceOptions extends ServiceOptions {
   private final String namespace;
   private final boolean force;
   private final Datastore datastore;
+  private final boolean normalizeDataset;
 
   public static class Builder extends ServiceOptions.Builder {
 
@@ -53,6 +54,7 @@ public static class Builder extends ServiceOptions.Builder {
     private String namespace;
     private boolean force;
     private Datastore datastore;
+    private boolean normalizeDataset = true;
 
     private Builder() {
     }
@@ -61,6 +63,9 @@ private Builder(DatastoreServiceOptions options) {
       super(options);
       dataset = options.dataset;
       force = options.force;
+      namespace = options.namespace;
+      datastore = options.datastore;
+      normalizeDataset = options.normalizeDataset;
     }
 
     @Override
@@ -87,10 +92,16 @@ public Builder force(boolean force) {
       this.force = force;
       return this;
     }
+
+    Builder normalizeDataset(boolean normalizeDataset) {
+      this.normalizeDataset = normalizeDataset;
+      return this;
+    }
   }
 
   private DatastoreServiceOptions(Builder builder) {
     super(builder);
+    normalizeDataset = builder.normalizeDataset;
     namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
     force = builder.force;
 
@@ -98,28 +109,33 @@ private DatastoreServiceOptions(Builder builder) {
     String tempDataset = firstNonNull(builder.dataset, defaultDataset());
     Datastore tempDatastore = firstNonNull(builder.datastore,
         defaultDatastore(tempDataset, host(), httpRequestInitializer()));
-    DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
-    DatastoreV1.Key key = DatastoreV1.Key.newBuilder()
-        .addPathElement(DatastoreV1.Key.PathElement.newBuilder().setKind("__foo__").setName("bar"))
-        .build();
-    requestPb.addKey(key);
-    try {
-      LookupResponse responsePb = tempDatastore.lookup(requestPb.build());
-      if (responsePb.getDeferredCount() > 0) {
-        key = responsePb.getDeferred(0);
-      } else {
-        Iterator combinedIter =
-            Iterables.concat(responsePb.getMissingList(), responsePb.getFoundList()).iterator();
-        key = combinedIter.next().getEntity().getKey();
-      }
-      dataset = key.getPartitionId().getDatasetId();
-      if (builder.datastore == null && !dataset.equals(tempDataset)) {
-        datastore = defaultDatastore(dataset, host(), httpRequestInitializer());
-      } else {
-        datastore = tempDatastore;
+    if (builder.normalizeDataset) {
+      DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
+      DatastoreV1.Key key = DatastoreV1.Key.newBuilder()
+          .addPathElement(DatastoreV1.Key.PathElement.newBuilder().setKind("__foo__").setName("bar"))
+          .build();
+      requestPb.addKey(key);
+      try {
+        LookupResponse responsePb = tempDatastore.lookup(requestPb.build());
+        if (responsePb.getDeferredCount() > 0) {
+          key = responsePb.getDeferred(0);
+        } else {
+          Iterator combinedIter =
+              Iterables.concat(responsePb.getMissingList(), responsePb.getFoundList()).iterator();
+          key = combinedIter.next().getEntity().getKey();
+        }
+        dataset = key.getPartitionId().getDatasetId();
+        if (builder.datastore == null && !dataset.equals(tempDataset)) {
+          datastore = defaultDatastore(dataset, host(), httpRequestInitializer());
+        } else {
+          datastore = tempDatastore;
+        }
+      } catch (DatastoreException e) {
+        throw DatastoreServiceException.translateAndThrow(e);
       }
-    } catch (DatastoreException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+    } else {
+      dataset = tempDataset;
+      datastore = tempDatastore;
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index 90be65882a44..645dd8eb3879 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -64,4 +64,8 @@ public Key allocateId() {
   protected PartialKey build() {
     return newKey();
   }
+
+  DatastoreService datastore() {
+    return service;
+  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
index 7d62ee88405f..decbbd547298 100644
--- a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -41,7 +41,7 @@ static final class Builder extends BaseEntity.Builder {
 
     private Key key;
 
-    private Builder() {
+    Builder() {
     }
 
     private Builder(ProjectionEntity entity) {
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
new file mode 100644
index 000000000000..dbe6c6284381
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -0,0 +1,67 @@
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.*;
+import static org.junit.Assert.assertTrue;
+
+import com.google.api.services.datastore.client.Datastore;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class DatastoreServiceOptionsTest {
+
+  private static final String DATASET = "dataset";
+  private Datastore datastore;
+  private DatastoreServiceOptions.Builder options;
+
+  @Before
+  public void setUp() throws IOException, InterruptedException {
+    datastore = EasyMock.createMock(Datastore.class);
+    options = DatastoreServiceOptions.builder()
+        .normalizeDataset(false)
+        .datastore(datastore)
+        .dataset(DATASET)
+        .host("http://localhost:" + LocalGcdHelper.PORT);
+  }
+
+  @Test
+  public void testDataset() throws Exception {
+    assertEquals(DATASET, options.build().dataset());
+  }
+
+  @Test
+  public void testHost() throws Exception {
+    assertEquals("http://localhost:" + LocalGcdHelper.PORT, options.build().host());
+  }
+
+  @Test
+  public void testNamespace() throws Exception {
+    assertNull(options.build().namespace());
+    assertEquals("ns1", options.namespace("ns1").build().namespace());
+  }
+
+  @Test
+  public void testForce() throws Exception {
+    assertFalse(options.build().force());
+    assertTrue(options.force(true).build().force());
+  }
+
+  @Test
+  public void testDatastore() throws Exception {
+    assertSame(datastore, options.build().datastore());
+  }
+
+  @Test
+  public void testToBuilder() throws Exception {
+    DatastoreServiceOptions original = options.namespace("ns1").force(true).build();
+    DatastoreServiceOptions copy = original.toBuilder().build();
+    assertEquals(original.dataset(), copy.dataset());
+    assertEquals(original.namespace(), copy.namespace());
+    assertEquals(original.host(), copy.host());
+    assertEquals(original.force(), copy.force());
+    assertEquals(original.retryParams(), copy.retryParams());
+    assertEquals(original.authConfig(), copy.authConfig());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
index 639c471a5155..00f48fd9fc1a 100644
--- a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
@@ -42,6 +42,11 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @Test(expected = IllegalArgumentException.class)
+  public void testIndexedNotAllowed() {
+    EntityValue.builder(CONTENT).indexed(true);
+  }
+
   @Test
   public void testBuilder() throws Exception {
     EntityValue.Builder builder = EntityValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
new file mode 100644
index 000000000000..143371f1eca6
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -0,0 +1,90 @@
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.google.api.services.datastore.DatastoreV1;
+import com.google.api.services.datastore.client.Datastore;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Iterator;
+
+public class KeyFactoryTest {
+
+  private static final String DATASET = "dataset";
+
+  private KeyFactory keyFactory;
+  private Datastore mock;
+
+  @Before
+  public void setUp() {
+    mock = EasyMock.createMock(Datastore.class);
+    DatastoreServiceOptions options =
+        DatastoreServiceOptions.builder().normalizeDataset(false).datastore(mock).dataset(DATASET).build();
+    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
+    keyFactory = new KeyFactory(datastore).kind("k");
+  }
+
+  @Test
+  public void testNewKey() throws Exception {
+    Key key = keyFactory.newKey(1);
+    verifyKey(key, 1L, null);
+    key = keyFactory.newKey("n");
+    verifyKey(key, "n", null);
+    PathElement p1 = PathElement.of("k1", "n");
+    PathElement p2 = PathElement.of("k2", 10);
+    key = keyFactory.namespace("ns").ancestors(p1, p2).newKey("k3");
+    verifyKey(key, "k3", "ns", p1, p2);
+  }
+
+  @Test
+  public void testNewPartialKey() throws Exception {
+    PartialKey key = keyFactory.newKey();
+    verifyPartialKey(key, null);
+    PathElement p1 = PathElement.of("k1", "n");
+    PathElement p2 = PathElement.of("k2", 10);
+    key = keyFactory.namespace("ns").ancestors(p1, p2).newKey();
+    verifyPartialKey(key, "ns", p1, p2);
+  }
+
+  @Test(expected = NullPointerException.class)
+  public void testNewPartialWithNoKind() {
+    new KeyFactory(keyFactory.datastore()).build();
+  }
+
+  private void verifyKey(Key key, String name, String namespace, PathElement... ancestors) {
+    assertEquals(name, key.name());
+    verifyPartialKey(key, namespace, ancestors);
+  }
+
+  private void verifyKey(Key key, Long id, String namespace, PathElement... ancestors) {
+    assertEquals(id, key.id());
+    verifyPartialKey(key, namespace, ancestors);
+  }
+
+  private void verifyPartialKey(PartialKey key, String namespace, PathElement... ancestors) {
+    assertEquals("k", key.kind());
+    assertEquals(DATASET, key.dataset());
+    assertEquals(namespace, key.namespace());
+    assertEquals(ancestors.length, key.ancestors().size());
+    Iterator iter = key.ancestors().iterator();
+    for (PathElement ancestor : ancestors) {
+      assertEquals(ancestor, iter.next());
+    }
+  }
+
+  @Test
+  public void testAllocateId() throws Exception {
+    PartialKey pk = keyFactory.newKey();
+    Key key = keyFactory.newKey(1);
+    DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder();
+    requestPb.addKey(pk.toPb());
+    DatastoreV1.AllocateIdsResponse.Builder responsePb = DatastoreV1.AllocateIdsResponse.newBuilder();
+    responsePb.addKey(key.toPb());
+    EasyMock.expect(mock.allocateIds(requestPb.build())).andReturn(responsePb.build());
+    EasyMock.replay(mock);
+    assertEquals(key, keyFactory.allocateId());
+    EasyMock.verify(mock);
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/ListValueTest.java b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
new file mode 100644
index 000000000000..7896457b99c7
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+
+import java.util.List;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ListValueTest {
+
+  private static final List> CONTENT = ImmutableList.of(NullValue.of(), StringValue.of("foo"));
+
+  @Test
+  public void testToBuilder() throws Exception {
+    ListValue value = ListValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    ListValue value = ListValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test(expected = DatastoreServiceException.class)
+  public void testIndexedCannotBeSpecified() {
+    ListValue.builder().indexed(false);
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    ListValue.Builder builder = ListValue.builder().set(CONTENT);
+    ListValue value = builder.meaning(1).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertFalse(value.hasIndexed());
+
+    builder = ListValue.builder();
+    for (Value v : CONTENT) {
+      builder.addValue(v);
+    }
+    assertEquals(CONTENT, builder.build().get());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
new file mode 100644
index 000000000000..f9c5dbe20647
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
@@ -0,0 +1,48 @@
+package com.google.gcloud.datastore;
+
+import org.junit.Test;
+
+import static junit.framework.TestCase.*;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ProjectionEntityTest {
+
+  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
+  private static final StringValue STRING_INDEX_VALUE = StringValue.builder("foo").meaning(18).build();
+  private static final BlobValue BLOB_VALUE = BlobValue.of(Blob.copyFrom(new byte[]{1}));
+  private static final DateTimeValue DATE_TIME_VALUE = DateTimeValue.of(DateTime.now());
+  private static final LongValue LONG_INDEX_VALUE =
+      LongValue.builder(DATE_TIME_VALUE.get().timestampMicroseconds()).meaning(18).build();
+  private static final ProjectionEntity ENTITY1 = new ProjectionEntity.Builder().key(KEY).set("a", "b").build();
+  private static final ProjectionEntity ENTITY2 = new ProjectionEntity.Builder()
+      .set("a", STRING_INDEX_VALUE)
+      .set("b", BLOB_VALUE)
+      .set("c", DATE_TIME_VALUE)
+      .set("d", LONG_INDEX_VALUE)
+      .build();
+
+  @Test
+  public void testHasKey() throws Exception {
+    assertTrue(ENTITY1.hasKey());
+    assertFalse(ENTITY2.hasKey());
+  }
+
+  @Test
+  public void testKey() throws Exception {
+    assertEquals(KEY, ENTITY1.key());
+    assertNull(ENTITY2.key());
+  }
+
+  @Test
+  public void testGetBlob() throws Exception {
+    assertArrayEquals(STRING_INDEX_VALUE.get().getBytes(), ENTITY2.getBlob("a").toByteArray());
+    assertEquals(BLOB_VALUE.get(), ENTITY2.getBlob("b"));
+  }
+
+  @Test
+  public void testGetDateTime() throws Exception {
+    assertEquals(DATE_TIME_VALUE.get(), ENTITY2.getDateTime("c"));
+    assertEquals(DATE_TIME_VALUE.get(), ENTITY2.getDateTime("d"));
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/RawValueTest.java b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
new file mode 100644
index 000000000000..7df4c7d544f5
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import com.google.api.services.datastore.DatastoreV1;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class RawValueTest {
+
+  private static final DatastoreV1.Value CONTENT = StringValue.of("hello").toPb();
+
+  @Test
+  public void testToBuilder() throws Exception {
+    RawValue value = RawValue.of(CONTENT);
+    assertEquals(value, value.toBuilder().build());
+  }
+
+  @Test
+  public void testOf() throws Exception {
+    RawValue value = RawValue.of(CONTENT);
+    assertEquals(CONTENT, value.get());
+    assertFalse(value.hasIndexed());
+    assertFalse(value.hasMeaning());
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+    RawValue.Builder builder = RawValue.builder(CONTENT);
+    RawValue value = builder.meaning(1).indexed(false).build();
+    assertEquals(CONTENT, value.get());
+    assertTrue(value.hasMeaning());
+    assertEquals(Integer.valueOf(1), value.meaning());
+    assertTrue(value.hasIndexed());
+    assertFalse(value.indexed());
+  }
+}

From 1c62eb09e6affdd73e82ca0ea5668fd4f579d736 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 19 Feb 2015 15:18:42 -0800
Subject: [PATCH 115/732] Fixes #6

---
 .../gcloud/datastore/DatastoreHelper.java     |   2 +-
 .../gcloud/datastore/DatastoreHelperTest.java | 160 ++++++++++++++++++
 2 files changed, 161 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index 0b0558e565fd..6b89b9f3ab2a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -109,7 +109,7 @@ public void delete(Key... key) {
    * Returns a new KeyFactory for this service
    */
   public KeyFactory newKeyFactory() {
-    return new KeyFactory(this);
+    return new KeyFactory(delegate);
   }
 
   /**
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
new file mode 100644
index 000000000000..531f5dd517da
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import com.google.api.services.datastore.client.Datastore;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+
+import org.junit.Test;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class DatastoreHelperTest {
+
+  @Test
+  public void testDelegate() throws Exception {
+    final Set methods = Sets.newHashSet(DatastoreService.class.getMethods());
+    final Map params = new HashMap<>();
+    InvocationHandler handler = new InvocationHandler() {
+      @Override
+      public Object invoke(Object o, Method method, Object[] objects) throws Throwable {
+        String methodName = Thread.currentThread().getStackTrace()[3].getMethodName();
+        if (method.getParameterTypes().length > 0) {
+          Class paramClass = method.getParameterTypes()[0];
+          methodName += paramClass.getSimpleName();
+          if (method.isVarArgs()) {
+            objects = (Object[]) objects[0];
+          }
+          assertArrayEquals(objects, params.get(methodName));
+        }
+        assertTrue(methods.remove(method));
+        return null;
+      }
+    };
+    PartialKey pKey1 = PartialKey.builder("ds", "k").build();
+    PartialKey pKey2 = PartialKey.builder("ds", "k").build();
+    PartialEntity pEntity1 = PartialEntity.builder(pKey1).build();
+    PartialEntity pEntity2 = PartialEntity.builder(pKey2).build();
+    Key key1 = Key.builder(pKey1, 1).build();
+    Key key2 = Key.builder(pKey1, "a").build();
+    Entity entity1 = Entity.builder(key1).build();
+    Entity entity2 = Entity.builder(key2).build();
+    ClassLoader cl = DatastoreService.class.getClassLoader();
+    Class[] interfaces = DatastoreHelper.class.getInterfaces();
+    DatastoreService delegate = (DatastoreService) Proxy.newProxyInstance(cl, interfaces, handler);
+    DatastoreHelper helper = DatastoreHelper.createFor(delegate);
+    params.put("getKey", new Object[] {key1});
+    helper.get(key1);
+    params.put("getKey[]", new Object[] {key1, key2});
+    helper.get(key1, key2);
+    params.put("addEntity[]", new Object[] {entity1, entity2});
+    helper.add(entity1, entity2);
+    params.put("updateEntity[]", new Object[] {entity1});
+    helper.update(entity1);
+    params.put("addPartialEntity", new Object[] {pEntity1});
+    helper.add(pEntity1);
+    params.put("addPartialEntity[]", new Object[] {pEntity2, entity1});
+    helper.add(pEntity2, entity1);
+    params.put("allocateIdPartialKey", new Object[] {pKey1});
+    helper.allocateId(pKey1);
+    params.put("allocateIdPartialKey[]", new Object[] {pKey1, pKey2});
+    helper.allocateId(pKey1, pKey2);
+    params.put("deleteKey[]", new Object[] {key1});
+    helper.delete(key1);
+    params.put("putEntity[]", new Object[] {entity1});
+    helper.put(entity1);
+    helper.options();
+    params.put("newBatchBatchOption[]", new Object[] {});
+    helper.newBatch();
+    params.put("newTransactionTransactionOption[]", new Object[] {});
+    helper.newTransaction();
+    Query query = createMock(Query.class);
+    params.put("runQuery", new Object[] {query});
+    helper.run(query);
+    assertTrue(methods.isEmpty());
+  }
+
+  @Test
+  public void testNewKeyFactory() throws Exception {
+    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    Datastore datastore = createStrictMock(Datastore.class);
+    DatastoreServiceOptions options =
+        DatastoreServiceOptions.builder().normalizeDataset(false).datastore(datastore)
+            .dataset("ds").build();
+    expect(datastoreService.options()).andReturn(options).atLeastOnce();
+    replay(datastore, datastoreService);
+    DatastoreHelper helper = DatastoreHelper.createFor(datastoreService);
+    KeyFactory keyFactory = helper.newKeyFactory();
+    assertSame(datastoreService, keyFactory.datastore());
+    verify(datastore, datastoreService);
+  }
+
+  @Test
+  public void testFetch() throws Exception {
+    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    PartialKey pKey1 = PartialKey.builder("ds", "k").build();
+    Key key1 = Key.builder(pKey1, 1).build();
+    Key key2 = Key.builder(pKey1, "a").build();
+    Entity entity1 = Entity.builder(key1).build();
+    Entity entity2 = Entity.builder(key2).build();
+    expect(datastoreService.get(key1, key2)).andReturn(Iterators.forArray(entity1, entity2)).once();
+    replay(datastoreService);
+    DatastoreHelper helper = DatastoreHelper.createFor(datastoreService);
+    List values = helper.fetch(key1, key2);
+    assertEquals(2, values.size());
+    assertEquals(entity1, values.get(0));
+    assertEquals(entity2, values.get(1));
+    verify(datastoreService);
+  }
+
+  @Test
+  public void testRunInTransaction() throws Exception {
+    final DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    final Transaction transaction = createStrictMock(Transaction.class);
+    expect(datastoreService.newTransaction()).andReturn(transaction).once();
+    expect(transaction.active()).andReturn(true).once();
+    expect(transaction.commit()).andReturn(null).once();
+    expect(transaction.active()).andReturn(false).once();
+    replay(datastoreService, transaction);
+    DatastoreHelper helper = DatastoreHelper.createFor(datastoreService);
+    helper.runInTransaction(new DatastoreHelper.RunInTransaction() {
+      @Override
+      public void run(DatastoreReaderWriter readerWriter) {
+        assertTrue(transaction.active());
+        assertSame(transaction, readerWriter);
+      }
+    });
+    verify(datastoreService, transaction);
+  }
+}

From e57ce121982d22bbeb050a36f6a3b0faab660a44 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 23 Feb 2015 10:35:36 -0800
Subject: [PATCH 116/732] update maven

---
 pom.xml | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f9a5dc52f8b9..2c0c2d82efaf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,12 +4,26 @@
   com.google.gcloud
   gcloud-java
   jar
-  0.0.1-SNAPSHOT
+  0.0.1
   GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   
     Java idiomatic client for Google Cloud Platform services.
   
+  
+    
+      ozarov
+      Arie Ozarov
+      ozarov@google.com
+      Google
+      
+        Developer
+      
+    
+  
+  
+    Google
+  
   
     scm:git:git@github.com:GoogleCloudPlatform/gcloud-java.git
     scm:git:git@github.com:GoogleCloudPlatform/gcloud-java.git
@@ -24,6 +38,16 @@
     https://github.com/GoogleCloudPlatform/gcloud-java/issues
     GitHub Issues
   
+  
+    
+      sonatype-nexus-snapshots
+      https://oss.sonatype.org/content/repositories/snapshots
+    
+    
+      sonatype-nexus-staging
+      https://oss.sonatype.org/service/local/staging/deploy/maven2/
+    
+  
   
     
       com.google.http-client
@@ -158,6 +182,57 @@
           UTF-8
         
       
+      
+        org.apache.maven.plugins
+        maven-source-plugin
+        2.4
+        
+          
+            attach-sources
+            
+              jar-no-fork
+            
+          
+        
+      
+      
+        org.apache.maven.plugins
+        maven-javadoc-plugin
+        2.10.1
+        
+          
+            attach-javadocs
+            
+              jar
+            
+          
+        
+      
+      
+        org.apache.maven.plugins
+        maven-gpg-plugin
+        1.5
+        
+          
+            sign-artifacts
+            verify
+            
+              sign
+            
+          
+        
+      
+      
+        org.sonatype.plugins
+	nexus-staging-maven-plugin
+	1.6.3
+	true
+	
+	  sonatype-nexus-staging
+	  https://oss.sonatype.org/
+	  true
+	
+      
       
         org.eluder.coveralls
         coveralls-maven-plugin

From e964aee4255c1c0f7701d151e5946ee066394832 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 23 Feb 2015 14:02:33 -0800
Subject: [PATCH 117/732] disable signing for CI

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5997f22ead37..ed1e40cca546 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,8 +7,8 @@ before_install:
 - mvn clean
 - git clone -b travis `git config --get remote.origin.url` target/travis
 - cp target/travis/settings.xml ~/.m2/settings.xml
-install: mvn install -DskipTests=true
-script: mvn verify
+install: mvn install -Dgpg.skip=true
+script: mvn verify -Dgpg.skip=true
 branches:
   only:
   - master

From 603745d4b49958a4d4e0654a2899ee74e53e43cc Mon Sep 17 00:00:00 2001
From: Arie 
Date: Tue, 24 Feb 2015 16:10:54 -0800
Subject: [PATCH 118/732] fix license URL

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 2c0c2d82efaf..b5b0ff32927c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,7 @@
   
     
       GCloud Java Software License
-      https://raw.github.com/GoogleCloudPlatform/gcloud-java/master/LICENSE.md
+      https://raw.githubusercontent.com/GoogleCloudPlatform/gcloud-java/master/LICENSE
     
   
   

From f973c8e6901f91bdb336625fb0b3a5fa047c5b04 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 25 Feb 2015 11:36:22 -0800
Subject: [PATCH 119/732] Add version info in site and jar

---
 pom.xml           | 18 ++++++++++++++++++
 src/site/site.xml |  1 +
 2 files changed, 19 insertions(+)

diff --git a/pom.xml b/pom.xml
index b5b0ff32927c..71e63e3ac9a4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -172,6 +172,22 @@
       
         maven-jar-plugin
         2.5
+        
+          
+            true
+            true
+            
+              true
+              true
+            
+            
+              ${project.artifactId}
+              ${project.groupId}
+              ${project.version}
+              ${buildNumber}
+            
+          
+        
       
       
         maven-compiler-plugin
@@ -295,6 +311,8 @@
 		    issue-tracking
 		    license
 		    scm
+		    distribution-management
+		    summary
                   
                 
               
diff --git a/src/site/site.xml b/src/site/site.xml
index db19e4a3877a..ec2726d99cde 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -22,6 +22,7 @@
     maven-fluido-skin
     1.3.1
   
+  
   
     
   

From b08c4590fe39321cc6dc2aacfe9b2f1be27228a1 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 26 Feb 2015 18:14:16 -0800
Subject: [PATCH 120/732] Use ServiceLoader for default RPC implementation

---
 .../java/com/google/gcloud/BaseService.java   |  1 +
 src/main/java/com/google/gcloud/Service.java  |  1 +
 .../com/google/gcloud/spi/DatastoreRpc.java   |  1 +
 .../gcloud/spi/DatastoreRpcFactory.java       |  1 +
 .../gcloud/spi/DefaultDatastoreRpc.java       |  1 +
 .../google/gcloud/spi/DefaultStorageRpc.java  |  1 +
 .../google/gcloud/spi/ServiceRpcFactory.java  |  1 +
 .../google/gcloud/spi/ServiceRpcProvider.java |  1 +
 .../com/google/gcloud/spi/StorageRpc.java     |  1 +
 .../google/gcloud/spi/StorageRpcFactory.java  |  1 +
 .../gcloud/datastore/DatastoreService.java    |  9 +-
 .../datastore/DatastoreServiceException.java  | 75 +++++++--------
 .../datastore/DatastoreServiceFactory.java    |  2 +-
 .../datastore/DatastoreServiceImpl.java       | 61 ++++++------
 .../datastore/DatastoreServiceOptions.java    | 96 +++++++++----------
 .../gcloud/datastore/StructuredQuery.java     |  2 +-
 .../com/google/gcloud/storage/BucketImpl.java |  1 +
 .../google/gcloud/storage/StorageService.java |  4 +-
 .../storage/StorageServiceException.java      |  1 +
 .../gcloud/storage/StorageServiceImpl.java    | 37 ++++---
 .../gcloud/storage/StorageServiceOptions.java | 39 ++++----
 .../gcloud/datastore/DatastoreHelperTest.java | 10 +-
 .../DatastoreServiceExceptionTest.java        |  1 +
 .../DatastoreServiceOptionsTest.java          | 16 ++--
 .../datastore/DatastoreServiceTest.java       |  6 +-
 .../gcloud/datastore/KeyFactoryTest.java      | 11 ++-
 .../gcloud/datastore/LocalGcdHelper.java      |  2 +-
 27 files changed, 200 insertions(+), 183 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/BaseService.java
 create mode 100644 src/main/java/com/google/gcloud/Service.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
 create mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
 create mode 100644 src/main/java/com/google/gcloud/storage/BucketImpl.java
 create mode 100644 src/main/java/com/google/gcloud/storage/StorageServiceException.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java

diff --git a/src/main/java/com/google/gcloud/BaseService.java b/src/main/java/com/google/gcloud/BaseService.java
new file mode 100644
index 000000000000..6672708785ac
--- /dev/null
+++ b/src/main/java/com/google/gcloud/BaseService.java
@@ -0,0 +1 @@
+package com.google.gcloud;

public abstract class BaseService implements Service {

  private final O options;

  protected BaseService(O options) {
    this.options = options;
  }

  @Override
  public O options() {
    return options;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/Service.java b/src/main/java/com/google/gcloud/Service.java
new file mode 100644
index 000000000000..b90255561185
--- /dev/null
+++ b/src/main/java/com/google/gcloud/Service.java
@@ -0,0 +1 @@
+package com.google.gcloud;

public interface Service {
  O options();
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
new file mode 100644
index 000000000000..84c1eb8bdded
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
@@ -0,0 +1 @@
+/*
 * Copyright 2013 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;

/**
 * Provides access to the remote Datastore service.
 */
public interface DatastoreRpc {

  public class DatastoreRpcException extends Exception {

    /**
     * The reason for the exception.
     *
     * @see Google Cloud Datastore error codes
     */
    public enum Reason {

      ABORTED(true, "Request aborted", 409),
      DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
      FAILED_PRECONDITION(false, "Invalid request", 412),
      INTERNAL(false, "Server returned an error", 500),
      INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
      PERMISSION_DENIED(false, "Unauthorized request", 403),
      RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
      UNAVAILABLE(true, "Could not reach service", 503);

      private final boolean retryable;
      private final String description;
      private final int httpStatus;

      private Reason(boolean retryable, String description, int httpStatus) {
        this.retryable = retryable;
        this.description = description;
        this.httpStatus = httpStatus;
      }

      public boolean retryable() {
        return retryable;
      }

      public String description() {
        return description;
      }

      public int httpStatus() {
        return httpStatus;
      }
    }

    private final String reason;
    private final int httpStatus;
    private final boolean retryable;

    public DatastoreRpcException(Reason reason) {
      this(reason.name(), reason.httpStatus, reason.retryable, reason.description);
    }

    public DatastoreRpcException(String reason, int httpStatus, boolean retryable, String message) {
      super(message);
      this.reason = reason;
      this.httpStatus = httpStatus;
      this.retryable = retryable;
    }

    public String reason() {
      return reason;
    }

    public int httpStatus() {
      return httpStatus;
    }

    public boolean retryable() {
      return retryable;
    }
  }

  AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException;

  BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException;

  CommitResponse commit(CommitRequest request) throws DatastoreRpcException;

  LookupResponse lookup(LookupRequest request) throws DatastoreRpcException;

  RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException;

  RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
new file mode 100644
index 000000000000..936221c24156
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
@@ -0,0 +1 @@
+package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.datastore.DatastoreServiceOptions;

public interface DatastoreRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
new file mode 100644
index 000000000000..e2dc9cb6db7b
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -0,0 +1 @@
+package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceOptions;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.dataset())
            .host(options.host())
            .initializer(options.httpTransport().createRequestFactory().getInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
new file mode 100644
index 000000000000..521b6f435711
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -0,0 +1 @@
+package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public Bucket bucket(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
new file mode 100644
index 000000000000..ee9fe8776518
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -0,0 +1 @@
+package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

public interface ServiceRpcFactory {

  S create(O options);
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
new file mode 100644
index 000000000000..f6d10dda5c9b
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
@@ -0,0 +1 @@
+package com.google.gcloud.com.google.gcloud.spi;

import com.google.common.collect.Iterables;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.storage.StorageServiceOptions;

import java.util.ServiceLoader;

public class ServiceRpcProvider {

  public static DatastoreRpc datastore(DatastoreServiceOptions options) {
    DatastoreRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(DatastoreRpcFactory.class), null);
    return factory == null ? new DefaultDatastoreRpc(options) : factory.create(options);
  }

  public static StorageRpc storage(StorageServiceOptions options) {
    StorageRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(StorageRpcFactory.class), null);
    return factory == null ? new DefaultStorageRpc(options) : factory.create(options);
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
new file mode 100644
index 000000000000..a895f696e16f
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
@@ -0,0 +1 @@
+package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
new file mode 100644
index 000000000000..c489346c3f30
--- /dev/null
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
@@ -0,0 +1 @@
+package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index 9770005dbc80..7d78192a5a1f 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -16,17 +16,14 @@
 
 package com.google.gcloud.datastore;
 
+import com.google.gcloud.Service;
+
 import java.util.List;
 
 /**
  * An interface for Google Cloud Datastore dataset.
  */
-public interface DatastoreService extends DatastoreReaderWriter {
-
-  /**
-   * Returns the {@code DatastoreServiceOptions} for this service.
-   */
-  DatastoreServiceOptions options();
+public interface DatastoreService extends Service, DatastoreReaderWriter {
 
   /**
    * Returns a new Datastore transaction.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index 9f1b4d6ad757..c365745863de 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -16,15 +16,12 @@
 
 package com.google.gcloud.datastore;
 
-import com.google.api.services.datastore.client.DatastoreException;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.RetryHelper;
 import com.google.gcloud.RetryHelper.RetryHelperException;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-import org.json.JSONTokener;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -44,40 +41,48 @@ public class DatastoreServiceException extends RuntimeException {
    */
   public enum Code {
 
-    ABORTED(true, "Request aborted", 409),
-    DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
-    UNAVAILABLE(true, "Could not reach service", 503),
-    FAILED_PRECONDITION(false, "Invalid request", 412),
-    INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
-    PERMISSION_DENIED(false, "Unauthorized request", 403),
+    ABORTED(Reason.ABORTED),
+    DEADLINE_EXCEEDED(Reason.DEADLINE_EXCEEDED),
+    UNAVAILABLE(Reason.UNAVAILABLE),
+    FAILED_PRECONDITION(Reason.FAILED_PRECONDITION),
+    INVALID_ARGUMENT(Reason.INVALID_ARGUMENT),
+    PERMISSION_DENIED(Reason.PERMISSION_DENIED),
     UNAUTHORIZED(false, "Unauthorized", 401),
-    RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
-    INTERNAL(false, "Server returned an error", 500),
+    INTERNAL(Reason.INTERNAL),
+    RESOURCE_EXHAUSTED(Reason.RESOURCE_EXHAUSTED),
     UNKNOWN(false, "Unknown failure", -1);
 
     private final boolean retriable;
-    private final String message;
-    private final int httpCode;
+    private final String description;
+    private final int httpStatus;
+
+    Code(Reason reason) {
+      this(reason.retryable(), reason.description(), reason.httpStatus());
+    }
 
-    Code(boolean retriable, String message, int httpCode) {
+    Code(boolean retriable, String description, int httpStatus) {
       this.retriable = retriable;
-      this.message = message;
-      this.httpCode = httpCode;
+      this.description = description;
+      this.httpStatus = httpStatus;
     }
 
-    public Integer httpCode() {
-      return httpCode;
+    public String description() {
+      return description;
+    }
+
+    public int httpStatus() {
+      return httpStatus;
     }
 
     /**
      * Returns {@code true} if this exception is transient and the same request could be retried.
      * For any retry it is highly recommended to apply an exponential backoff.
      */
-    public boolean isRetriable() {
+    public boolean retriable() {
       return retriable;
     }
 
-    DatastoreServiceException translate(DatastoreException exception, String message) {
+    DatastoreServiceException translate(DatastoreRpcException exception, String message) {
       return new DatastoreServiceException(this, message, exception);
     }
   }
@@ -87,14 +92,14 @@ DatastoreServiceException translate(DatastoreException exception, String message
     Map httpCodes = new HashMap<>();
     for (Code code : Code.values()) {
       builder.put(code.name(), code);
-      httpCodes.put(code.httpCode(), code);
+      httpCodes.put(code.httpStatus(), code);
     }
     REASON_TO_CODE = builder.build();
     HTTP_TO_CODE = ImmutableMap.copyOf(httpCodes);
   }
 
   public DatastoreServiceException(Code code, String message, Exception cause) {
-    super(MoreObjects.firstNonNull(message, code.message), cause);
+    super(MoreObjects.firstNonNull(message, code.description), cause);
     this.code = code;
   }
 
@@ -110,8 +115,8 @@ public Code code() {
   }
 
   static DatastoreServiceException translateAndThrow(RetryHelperException ex) {
-    if (ex.getCause() instanceof DatastoreException) {
-      return translateAndThrow((DatastoreException) ex.getCause());
+    if (ex.getCause() instanceof DatastoreRpcException) {
+      return translateAndThrow((DatastoreRpcException) ex.getCause());
     }
     if (ex instanceof RetryHelper.RetryInterruptedException) {
       RetryHelper.RetryInterruptedException.propagate();
@@ -125,27 +130,15 @@ static DatastoreServiceException translateAndThrow(RetryHelperException ex) {
    *
    * @throws DatastoreServiceException every time
    */
-  static DatastoreServiceException translateAndThrow(DatastoreException exception) {
+  static DatastoreServiceException translateAndThrow(DatastoreRpcException exception) {
     String message = exception.getMessage();
-    String reason = "";
-    if (message != null) {
-      try {
-        JSONObject json = new JSONObject(new JSONTokener(message));
-        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
-        reason = error.getString("reason");
-        message = error.getString("message");
-      } catch (JSONException ignore) {
-        // ignore - will be converted to unknown
-      }
-    }
-    Code code = REASON_TO_CODE.get(reason);
+    Code code = REASON_TO_CODE.get(exception.reason());
     if (code == null) {
-      code = MoreObjects.firstNonNull(HTTP_TO_CODE.get(exception.getCode()), Code.UNKNOWN);
+      code = MoreObjects.firstNonNull(HTTP_TO_CODE.get(exception.httpStatus()), Code.UNKNOWN);
     }
     throw code.translate(exception, message);
   }
 
-
   /**
    * Throw a DatastoreServiceException with {@code FAILED_PRECONDITION} code and the {@code message}
    * in a nested exception.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
index ca87e9cdad9b..ca8dfa7e2982 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
@@ -23,7 +23,7 @@ public abstract class DatastoreServiceFactory {
   private static final DatastoreServiceFactory INSTANCE = new DatastoreServiceFactory() {
       @Override
       public DatastoreService get(DatastoreServiceOptions options) {
-        return new DatastoreServiceImpl(options, options.datastore());
+        return new DatastoreServiceImpl(options);
       }
     };
 
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index c100e45e548a..231624445e76 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -17,8 +17,6 @@
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.api.services.datastore.client.Datastore;
-import com.google.api.services.datastore.client.DatastoreException;
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
@@ -26,10 +24,14 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Sets;
+import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
+import com.google.gcloud.ExceptionHandler.Interceptor;
 import com.google.gcloud.RetryHelper;
 import com.google.gcloud.RetryHelper.RetryHelperException;
 import com.google.gcloud.RetryParams;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
 import com.google.protobuf.ByteString;
 
 import java.util.Arrays;
@@ -43,10 +45,11 @@
 import java.util.concurrent.Callable;
 
 
-final class DatastoreServiceImpl implements DatastoreService {
+final class DatastoreServiceImpl extends BaseService
+    implements DatastoreService {
 
-  private static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
-      new ExceptionHandler.Interceptor() {
+  private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new Interceptor() {
 
         private static final long serialVersionUID = 6911242958397733203L;
 
@@ -58,33 +61,25 @@ public RetryResult afterEval(Exception exception, RetryResult retryResult) {
         @Override
         public RetryResult beforeEval(Exception exception) {
           if (exception instanceof DatastoreServiceException) {
-            boolean isRetriable = ((DatastoreServiceException) exception).code().isRetriable();
-            return isRetriable
-                ? ExceptionHandler.Interceptor.RetryResult.RETRY
-                : ExceptionHandler.Interceptor.RetryResult.ABORT;
+            boolean retriable = ((DatastoreServiceException) exception).code().retriable();
+            return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
           }
           return null;
         }
       };
   private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
-      .abortOn(RuntimeException.class, DatastoreException.class)
+      .abortOn(RuntimeException.class, DatastoreRpcException.class)
       .interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
 
-  private final DatastoreServiceOptions options;
-  private final Datastore datastore;
+  private final DatastoreRpc datastoreRpc;
   private final RetryParams retryParams;
 
-  DatastoreServiceImpl(DatastoreServiceOptions options, Datastore datastore) {
-    this.options = options;
-    this.datastore = datastore;
+  DatastoreServiceImpl(DatastoreServiceOptions options) {
+    super(options);
+    this.datastoreRpc = options.datastoreRpc();
     retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
   }
 
-  @Override
-  public DatastoreServiceOptions options() {
-    return options;
-  }
-
   @Override
   public Batch newBatch(BatchOption... options) {
     return new BatchImpl(this, options);
@@ -107,8 +102,8 @@  QueryResult run(DatastoreV1.ReadOptions readOptionsPb, Query query) {
   DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestPb) {
     try {
       return RetryHelper.runWithRetries(new Callable() {
-        @Override public DatastoreV1.RunQueryResponse call() throws DatastoreException {
-          return datastore.runQuery(requestPb);
+        @Override public DatastoreV1.RunQueryResponse call() throws DatastoreRpcException {
+          return datastoreRpc.runQuery(requestPb);
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
@@ -146,8 +141,8 @@ public Key apply(DatastoreV1.Key keyPb) {
   DatastoreV1.AllocateIdsResponse allocateIds(final DatastoreV1.AllocateIdsRequest requestPb) {
     try {
       return RetryHelper.runWithRetries(new Callable() {
-        @Override public DatastoreV1.AllocateIdsResponse call() throws DatastoreException {
-          return datastore.allocateIds(requestPb);
+        @Override public DatastoreV1.AllocateIdsResponse call() throws DatastoreRpcException {
+          return datastoreRpc.allocateIds(requestPb);
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
@@ -269,8 +264,8 @@ protected Entity computeNext() {
   DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) {
     try {
       return RetryHelper.runWithRetries(new Callable() {
-        @Override public DatastoreV1.LookupResponse call() throws DatastoreException {
-          return datastore.lookup(requestPb);
+        @Override public DatastoreV1.LookupResponse call() throws DatastoreRpcException {
+          return datastoreRpc.lookup(requestPb);
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
@@ -337,7 +332,7 @@ public void delete(Key... keys) {
   }
 
   private DatastoreV1.CommitResponse commitMutation(DatastoreV1.Mutation.Builder mutationPb) {
-    if (options.force()) {
+    if (options().force()) {
       mutationPb.setForce(true);
     }
     DatastoreV1.CommitRequest.Builder requestPb = DatastoreV1.CommitRequest.newBuilder();
@@ -349,8 +344,8 @@ private DatastoreV1.CommitResponse commitMutation(DatastoreV1.Mutation.Builder m
   DatastoreV1.CommitResponse commit(final DatastoreV1.CommitRequest requestPb) {
     try {
       return RetryHelper.runWithRetries(new Callable() {
-        @Override public DatastoreV1.CommitResponse call() throws DatastoreException {
-          return datastore.commit(requestPb);
+        @Override public DatastoreV1.CommitResponse call() throws DatastoreRpcException {
+          return datastoreRpc.commit(requestPb);
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
@@ -367,8 +362,8 @@ DatastoreV1.BeginTransactionResponse beginTransaction(
     try {
       return RetryHelper.runWithRetries(new Callable() {
         @Override
-        public DatastoreV1.BeginTransactionResponse call() throws DatastoreException {
-          return datastore.beginTransaction(requestPb);
+        public DatastoreV1.BeginTransactionResponse call() throws DatastoreRpcException {
+          return datastoreRpc.beginTransaction(requestPb);
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
@@ -385,8 +380,8 @@ void rollbackTransaction(ByteString transaction) {
   void rollback(final DatastoreV1.RollbackRequest requestPb) {
     try {
       RetryHelper.runWithRetries(new Callable() {
-        @Override public Void call() throws DatastoreException {
-          datastore.rollback(requestPb);
+        @Override public Void call() throws DatastoreRpcException {
+          datastoreRpc.rollback(requestPb);
           return null;
         }
       }, retryParams, EXCEPTION_HANDLER);
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index f3d90ab38597..f57b222b6742 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -20,17 +20,15 @@
 import static com.google.gcloud.datastore.Validator.validateDataset;
 import static com.google.gcloud.datastore.Validator.validateNamespace;
 
-import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.api.services.datastore.DatastoreV1.EntityResult;
 import com.google.api.services.datastore.DatastoreV1.LookupResponse;
-import com.google.api.services.datastore.client.Datastore;
-import com.google.api.services.datastore.client.DatastoreException;
-import com.google.api.services.datastore.client.DatastoreFactory;
-import com.google.api.services.datastore.client.DatastoreOptions;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.gcloud.ServiceOptions;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
+import com.google.gcloud.com.google.gcloud.spi.ServiceRpcProvider;
 
 import java.lang.reflect.Method;
 import java.util.Iterator;
@@ -45,15 +43,16 @@ public class DatastoreServiceOptions extends ServiceOptions {
   private final String dataset;
   private final String namespace;
   private final boolean force;
-  private final Datastore datastore;
+  private final DatastoreRpc datastoreRpc;
   private final boolean normalizeDataset;
+  private final boolean defaultDatastoreRpc;
 
   public static class Builder extends ServiceOptions.Builder {
 
     private String dataset;
     private String namespace;
     private boolean force;
-    private Datastore datastore;
+    private DatastoreRpc datastoreRpc;
     private boolean normalizeDataset = true;
 
     private Builder() {
@@ -64,17 +63,18 @@ private Builder(DatastoreServiceOptions options) {
       dataset = options.dataset;
       force = options.force;
       namespace = options.namespace;
-      datastore = options.datastore;
+      datastoreRpc = options.datastoreRpc;
       normalizeDataset = options.normalizeDataset;
     }
 
     @Override
     public DatastoreServiceOptions build() {
-      return new DatastoreServiceOptions(this);
+      DatastoreServiceOptions options = new DatastoreServiceOptions(this);
+      return normalizeDataset ? options.normalize() : options;
     }
 
-    public Builder datastore(Datastore datastore) {
-      this.datastore = datastore;
+    public Builder datastoreRpc(DatastoreRpc datastoreRpc) {
+      this.datastoreRpc = datastoreRpc;
       return this;
     }
 
@@ -104,49 +104,41 @@ private DatastoreServiceOptions(Builder builder) {
     normalizeDataset = builder.normalizeDataset;
     namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
     force = builder.force;
+    dataset = firstNonNull(builder.dataset, defaultDataset());
+    datastoreRpc = firstNonNull(builder.datastoreRpc, ServiceRpcProvider.datastore(this));
+    defaultDatastoreRpc = builder.datastoreRpc == null;
+  }
 
-    // Replace provided dataset with full dataset (s~xxx, e~xxx,...)
-    String tempDataset = firstNonNull(builder.dataset, defaultDataset());
-    Datastore tempDatastore = firstNonNull(builder.datastore,
-        defaultDatastore(tempDataset, host(), httpRequestInitializer()));
-    if (builder.normalizeDataset) {
-      DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
-      DatastoreV1.Key key = DatastoreV1.Key.newBuilder()
-          .addPathElement(DatastoreV1.Key.PathElement.newBuilder().setKind("__foo__").setName("bar"))
-          .build();
-      requestPb.addKey(key);
-      try {
-        LookupResponse responsePb = tempDatastore.lookup(requestPb.build());
-        if (responsePb.getDeferredCount() > 0) {
-          key = responsePb.getDeferred(0);
-        } else {
-          Iterator combinedIter =
-              Iterables.concat(responsePb.getMissingList(), responsePb.getFoundList()).iterator();
-          key = combinedIter.next().getEntity().getKey();
-        }
-        dataset = key.getPartitionId().getDatasetId();
-        if (builder.datastore == null && !dataset.equals(tempDataset)) {
-          datastore = defaultDatastore(dataset, host(), httpRequestInitializer());
-        } else {
-          datastore = tempDatastore;
-        }
-      } catch (DatastoreException e) {
-        throw DatastoreServiceException.translateAndThrow(e);
-      }
-    } else {
-      dataset = tempDataset;
-      datastore = tempDatastore;
+  private DatastoreServiceOptions normalize() {
+    if (!normalizeDataset) {
+      return this;
     }
-  }
 
-  private static Datastore defaultDatastore(
-      String dataset, String host, HttpRequestInitializer initializer) {
-    DatastoreOptions options = new DatastoreOptions.Builder()
-        .dataset(dataset)
-        .host(host)
-        .initializer(initializer)
+    Builder builder = toBuilder();
+    builder.normalizeDataset(false);
+    // Replace provided dataset with full dataset (s~xxx, e~xxx,...)
+    DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
+    DatastoreV1.Key key = DatastoreV1.Key.newBuilder()
+        .addPathElement(DatastoreV1.Key.PathElement.newBuilder().setKind("__foo__").setName("bar"))
         .build();
-    return DatastoreFactory.get().create(options);
+    requestPb.addKey(key);
+    try {
+      LookupResponse responsePb = datastoreRpc.lookup(requestPb.build());
+      if (responsePb.getDeferredCount() > 0) {
+        key = responsePb.getDeferred(0);
+      } else {
+        Iterator combinedIter =
+            Iterables.concat(responsePb.getMissingList(), responsePb.getFoundList()).iterator();
+        key = combinedIter.next().getEntity().getKey();
+      }
+      builder.dataset(key.getPartitionId().getDatasetId());
+      if (defaultDatastoreRpc && !dataset.equals(builder.dataset)) {
+        builder.datastoreRpc(ServiceRpcProvider.datastore(builder.build()));
+      }
+      return new DatastoreServiceOptions(builder);
+    } catch (DatastoreRpcException e) {
+      throw DatastoreServiceException.translateAndThrow(e);
+    }
   }
 
   private static String defaultDataset() {
@@ -193,8 +185,8 @@ public Builder toBuilder() {
     return new Builder(this);
   }
 
-  public Datastore datastore() {
-    return datastore;
+  public DatastoreRpc datastoreRpc() {
+    return datastoreRpc;
   }
 
   public static Builder builder() {
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index d86086f53088..03372fe3d96f 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -16,7 +16,7 @@
 
 package com.google.gcloud.datastore;
 
-import static com.google.api.client.util.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.gcloud.datastore.BlobValue.of;
 import static com.google.gcloud.datastore.BooleanValue.of;
 import static com.google.gcloud.datastore.DateTimeValue.of;
diff --git a/src/main/java/com/google/gcloud/storage/BucketImpl.java b/src/main/java/com/google/gcloud/storage/BucketImpl.java
new file mode 100644
index 000000000000..8509b63655e0
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BucketImpl.java
@@ -0,0 +1 @@
+package com.google.gcloud.storage;

import com.google.gcloud.storage.StorageObject.Key;

import java.util.Iterator;

public class BucketImpl implements Bucket {

  private final StorageServiceImpl storageService;
  private final com.google.api.services.storage.model.Bucket model;

  BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) {
    this.storageService = storageService;
    this.model = model;
  }

  @Override
  public String id() {
    return null;
  }

  @Override
  public String name() {
    return null;
  }

  @Override
  public Cors cors() {
    return null;
  }

  @Override
  public Acl acl() {
    return null;
  }

  @Override
  public Acl defaultObjectAcl() {
    return null;
  }

  @Override
  public void updateCors(Cors cors) {

  }

  @Override
  public void updateAcl(Acl acl) {

  }

  @Override
  public void updateDefaultObjectAcl() {

  }

  @Override
  public void delete(String... objectName) {

  }

  @Override
  public void compose(Iterable sourceObjectNames, String destObjectName) {

  }

  @Override
  public void copy(String sourceObjectName, Key destObjectKey) {

  }

  @Override
  public StorageObject get(String objectName) {
    return null;
  }

  @Override
  public Iterator get(String... objectName) {
    return null;
  }

  @Override
  public InputChannel getInputChannel(String ObjectName) {
    return null;
  }

  @Override
  public OutputChannel getOutputChannel(String ObjectName) {
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 313a01f82258..3fd0b6b3fa1b 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -16,7 +16,9 @@
 
 package com.google.gcloud.storage;
 
-public interface StorageService {
+import com.google.gcloud.Service;
+
+public interface StorageService extends Service {
 
   Iterable listBuckets();
 
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
new file mode 100644
index 000000000000..9779b20aabcf
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
@@ -0,0 +1 @@
+package com.google.gcloud.storage;

import java.io.IOException;

public class StorageServiceException extends RuntimeException {

  StorageServiceException(IOException ex) {
    super(ex);
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 45a347d27c6d..4c84d6f9d67b 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,29 +16,42 @@
 
 package com.google.gcloud.storage;
 
-import com.google.api.services.storage.Storage;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.gcloud.BaseService;
+import com.google.gcloud.com.google.gcloud.spi.StorageRpc;
 
-final class StorageServiceImpl implements StorageService {
+import java.io.IOException;
 
-  private final StorageServiceOptions options;
-  private final Storage storage;
+final class StorageServiceImpl extends BaseService implements StorageService {
+
+  private final StorageRpc storageRpc;
 
   StorageServiceImpl(StorageServiceOptions options) {
-    this.options = options;
-    storage = options.getStorage();
+    super(options);
+    storageRpc = options.storageRpc();
   }
 
   @Override
   public Iterable listBuckets() {
-    // TODO Auto-generated method stub
-    return null;
+    try {
+      return Iterables.transform(storageRpc.buckets(),
+          new Function() {
+            @Override public Bucket apply(com.google.api.services.storage.model.Bucket model) {
+              return new BucketImpl(StorageServiceImpl.this, model);
+            }
+          });
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
   public Bucket getBucket(String bucket) {
-    // TODO Auto-generated method stub
-    return null;
+    try {
+      return new BucketImpl(this, storageRpc.bucket(bucket));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
-
-
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index a4a8e60adf94..764c0df4fcb3 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -16,12 +16,12 @@
 
 package com.google.gcloud.storage;
 
-import com.google.api.client.json.jackson.JacksonFactory;
-import com.google.api.services.storage.Storage;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import com.google.gcloud.ServiceOptions;
+import com.google.gcloud.com.google.gcloud.spi.ServiceRpcProvider;
+import com.google.gcloud.com.google.gcloud.spi.StorageRpc;
 
 import java.util.Set;
 
@@ -31,20 +31,15 @@ public class StorageServiceOptions extends ServiceOptions {
   private static final Set SCOPES = ImmutableSet.of(GCS_SCOPE);
   private static final String DEFAULT_PATH_DELIMITER = "/";
 
+  private final StorageRpc storageRpc;
   private final String project;
   private final String pathDelimiter;
 
-  private StorageServiceOptions(Builder builder) {
-    super(builder);
-    pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
-    project = builder.project != null ? builder.project :  getAppEngineProjectId();
-    Preconditions.checkArgument(project != null, "Missing required project id");
-  }
-
   public static class Builder extends ServiceOptions.Builder {
 
     private String project;
     private String pathDelimiter;
+    private StorageRpc storageRpc;
 
     private Builder() {
     }
@@ -63,20 +58,36 @@ public Builder pathDelimiter(String pathDelimiter) {
       return this;
     }
 
+    public Builder storageRpc(StorageRpc storageRpc) {
+      this.storageRpc = storageRpc;
+      return this;
+    }
+
     @Override
     public StorageServiceOptions build() {
       return new StorageServiceOptions(this);
     }
   }
 
+  private StorageServiceOptions(Builder builder) {
+    super(builder);
+    pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
+    project = builder.project != null ? builder.project :  getAppEngineProjectId();
+    Preconditions.checkArgument(project != null, "Missing required project id");
+    storageRpc = MoreObjects.firstNonNull(builder.storageRpc, ServiceRpcProvider.storage(this));
+  }
+
   @Override
   protected Set scopes() {
     return SCOPES;
   }
 
-  Storage getStorage() {
-    return new Storage.Builder(httpTransport(), new JacksonFactory(), httpRequestInitializer())
-        .build();
+  public StorageRpc storageRpc() {
+    return storageRpc;
+  }
+
+  public String project() {
+    return project;
   }
 
   public String pathDelimiter() {
@@ -91,8 +102,4 @@ public Builder toBuilder() {
   public static Builder builder() {
     return new Builder();
   }
-
-  public static Builder builder(StorageServiceOptions options) {
-    return new Builder(options);
-  }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 531f5dd517da..915d792a82fa 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -26,9 +26,9 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
 
-import com.google.api.services.datastore.client.Datastore;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Sets;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
 
 import org.junit.Test;
 
@@ -108,16 +108,16 @@ public Object invoke(Object o, Method method, Object[] objects) throws Throwable
   @Test
   public void testNewKeyFactory() throws Exception {
     DatastoreService datastoreService = createStrictMock(DatastoreService.class);
-    Datastore datastore = createStrictMock(Datastore.class);
+    DatastoreRpc datastoreRpc = createStrictMock(DatastoreRpc.class);
     DatastoreServiceOptions options =
-        DatastoreServiceOptions.builder().normalizeDataset(false).datastore(datastore)
+        DatastoreServiceOptions.builder().normalizeDataset(false).datastoreRpc(datastoreRpc)
             .dataset("ds").build();
     expect(datastoreService.options()).andReturn(options).atLeastOnce();
-    replay(datastore, datastoreService);
+    replay(datastoreRpc, datastoreService);
     DatastoreHelper helper = DatastoreHelper.createFor(datastoreService);
     KeyFactory keyFactory = helper.newKeyFactory();
     assertSame(datastoreService, keyFactory.datastore());
-    verify(datastore, datastoreService);
+    verify(datastoreRpc, datastoreService);
   }
 
   @Test
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
new file mode 100644
index 000000000000..4b2404358082
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
@@ -0,0 +1 @@
+package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retriable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
index dbe6c6284381..d54d87814b60 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -1,9 +1,13 @@
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.*;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertNull;
+import static junit.framework.TestCase.assertSame;
 import static org.junit.Assert.assertTrue;
 
-import com.google.api.services.datastore.client.Datastore;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
+
 import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
@@ -13,15 +17,15 @@
 public class DatastoreServiceOptionsTest {
 
   private static final String DATASET = "dataset";
-  private Datastore datastore;
+  private DatastoreRpc datastoreRpc;
   private DatastoreServiceOptions.Builder options;
 
   @Before
   public void setUp() throws IOException, InterruptedException {
-    datastore = EasyMock.createMock(Datastore.class);
+    datastoreRpc = EasyMock.createMock(DatastoreRpc.class);
     options = DatastoreServiceOptions.builder()
         .normalizeDataset(false)
-        .datastore(datastore)
+        .datastoreRpc(datastoreRpc)
         .dataset(DATASET)
         .host("http://localhost:" + LocalGcdHelper.PORT);
   }
@@ -50,7 +54,7 @@ public void testForce() throws Exception {
 
   @Test
   public void testDatastore() throws Exception {
-    assertSame(datastore, options.build().datastore());
+    assertSame(datastoreRpc, options.build().datastoreRpc());
   }
 
   @Test
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 10537734ef16..c5c5613a4bbc 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -24,11 +24,11 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import com.google.api.services.datastore.client.DatastoreException;
 import com.google.gcloud.datastore.Query.Type;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
+
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -313,7 +313,7 @@ public void testNewBatch() {
   }
 
   @Test
-  public void testRunGqlQueryNoCasting() throws DatastoreException {
+  public void testRunGqlQueryNoCasting() {
     Query query1 = GqlQuery.builder(Type.FULL, "select * from " + KIND1).build();
     QueryResult results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
@@ -384,7 +384,7 @@ public void testRunGqlQueryWithCasting() {
   }
 
   @Test
-  public void testRunStructuredQuery() throws DatastoreException {
+  public void testRunStructuredQuery() {
     StructuredQuery query =
         StructuredQuery.builder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
     QueryResult results1 = datastore.run(query);
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index 143371f1eca6..eb2d1567cb29 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -3,7 +3,8 @@
 import static junit.framework.TestCase.assertEquals;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.api.services.datastore.client.Datastore;
+import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
+
 import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
@@ -15,13 +16,13 @@ public class KeyFactoryTest {
   private static final String DATASET = "dataset";
 
   private KeyFactory keyFactory;
-  private Datastore mock;
+  private DatastoreRpc mock;
 
   @Before
   public void setUp() {
-    mock = EasyMock.createMock(Datastore.class);
-    DatastoreServiceOptions options =
-        DatastoreServiceOptions.builder().normalizeDataset(false).datastore(mock).dataset(DATASET).build();
+    mock = EasyMock.createMock(DatastoreRpc.class);
+    DatastoreServiceOptions options = DatastoreServiceOptions.builder().normalizeDataset(false)
+        .datastoreRpc(mock).dataset(DATASET).build();
     DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
     keyFactory = new KeyFactory(datastore).kind("k");
   }
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 5a71442b8a6b..521089f78d08 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -18,7 +18,7 @@
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.api.client.util.Strings;
+import com.google.common.base.Strings;
 
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;

From cce26a1f95c9ee3e0842a34d5d1ea5f728f86887 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 26 Feb 2015 18:19:31 -0800
Subject: [PATCH 121/732] update imports

---
 .../java/com/google/gcloud/datastore/ListValueTest.java  | 9 +++++----
 .../google/gcloud/datastore/ProjectionEntityTest.java    | 8 +++++---
 .../java/com/google/gcloud/datastore/RawValueTest.java   | 7 ++++---
 src/test/java/com/google/gcloud/datastore/ValueTest.java | 1 +
 4 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/test/java/com/google/gcloud/datastore/ListValueTest.java b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
index 7896457b99c7..4a0168fb2b1e 100644
--- a/src/test/java/com/google/gcloud/datastore/ListValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
@@ -16,15 +16,16 @@
 
 package com.google.gcloud.datastore;
 
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import com.google.common.collect.ImmutableList;
+
 import org.junit.Test;
 
 import java.util.List;
 
-import static junit.framework.TestCase.assertFalse;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
 public class ListValueTest {
 
   private static final List> CONTENT = ImmutableList.of(NullValue.of(), StringValue.of("foo"));
diff --git a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
index f9c5dbe20647..601f73659fbd 100644
--- a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
@@ -1,11 +1,13 @@
 package com.google.gcloud.datastore;
 
-import org.junit.Test;
-
-import static junit.framework.TestCase.*;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertNull;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertTrue;
 
+import org.junit.Test;
+
 public class ProjectionEntityTest {
 
   private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
diff --git a/src/test/java/com/google/gcloud/datastore/RawValueTest.java b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
index 7df4c7d544f5..48cc88c20e63 100644
--- a/src/test/java/com/google/gcloud/datastore/RawValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
@@ -16,13 +16,14 @@
 
 package com.google.gcloud.datastore;
 
-import com.google.api.services.datastore.DatastoreV1;
-import org.junit.Test;
-
 import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import com.google.api.services.datastore.DatastoreV1;
+
+import org.junit.Test;
+
 public class RawValueTest {
 
   private static final DatastoreV1.Value CONTENT = StringValue.of("hello").toPb();
diff --git a/src/test/java/com/google/gcloud/datastore/ValueTest.java b/src/test/java/com/google/gcloud/datastore/ValueTest.java
index 04364283ecac..161910126156 100644
--- a/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -25,6 +25,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.primitives.Primitives;
 import com.google.gcloud.datastore.Value.Type;
+
 import org.junit.Before;
 import org.junit.Test;
 

From 726f4917c2a1dbfecdcf02697776fcf693cf1514 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 26 Feb 2015 18:23:56 -0800
Subject: [PATCH 122/732] update copyright

---
 src/main/java/com/google/gcloud/BaseService.java |  2 +-
 src/main/java/com/google/gcloud/Service.java     |  2 +-
 .../com/google/gcloud/spi/DatastoreRpc.java      |  2 +-
 .../google/gcloud/spi/DatastoreRpcFactory.java   |  2 +-
 .../google/gcloud/spi/DefaultDatastoreRpc.java   |  2 +-
 .../com/google/gcloud/spi/DefaultStorageRpc.java |  2 +-
 .../com/google/gcloud/spi/ServiceRpcFactory.java |  2 +-
 .../google/gcloud/spi/ServiceRpcProvider.java    |  2 +-
 .../gcloud/com/google/gcloud/spi/StorageRpc.java |  2 +-
 .../com/google/gcloud/spi/StorageRpcFactory.java |  2 +-
 .../com/google/gcloud/storage/BucketImpl.java    |  2 +-
 .../gcloud/storage/StorageServiceException.java  |  2 +-
 .../datastore/DatastoreServiceExceptionTest.java |  2 +-
 .../datastore/DatastoreServiceOptionsTest.java   | 16 ++++++++++++++++
 .../google/gcloud/datastore/KeyFactoryTest.java  | 16 ++++++++++++++++
 .../gcloud/datastore/ProjectionEntityTest.java   | 16 ++++++++++++++++
 16 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/src/main/java/com/google/gcloud/BaseService.java b/src/main/java/com/google/gcloud/BaseService.java
index 6672708785ac..4d96ac5b86e6 100644
--- a/src/main/java/com/google/gcloud/BaseService.java
+++ b/src/main/java/com/google/gcloud/BaseService.java
@@ -1 +1 @@
-package com.google.gcloud;

public abstract class BaseService implements Service {

  private final O options;

  protected BaseService(O options) {
    this.options = options;
  }

  @Override
  public O options() {
    return options;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud;

public abstract class BaseService implements Service {

  private final O options;

  protected BaseService(O options) {
    this.options = options;
  }

  @Override
  public O options() {
    return options;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/Service.java b/src/main/java/com/google/gcloud/Service.java
index b90255561185..02d810186bfc 100644
--- a/src/main/java/com/google/gcloud/Service.java
+++ b/src/main/java/com/google/gcloud/Service.java
@@ -1 +1 @@
-package com.google.gcloud;

public interface Service {
  O options();
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud;

public interface Service {
  O options();
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
index 84c1eb8bdded..45b2f43efe44 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2013 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;

/**
 * Provides access to the remote Datastore service.
 */
public interface DatastoreRpc {

  public class DatastoreRpcException extends Exception {

    /**
     * The reason for the exception.
     *
     * @see Google Cloud Datastore error codes
     */
    public enum Reason {

      ABORTED(true, "Request aborted", 409),
      DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
      FAILED_PRECONDITION(false, "Invalid request", 412),
      INTERNAL(false, "Server returned an error", 500),
      INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
      PERMISSION_DENIED(false, "Unauthorized request", 403),
      RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
      UNAVAILABLE(true, "Could not reach service", 503);

      private final boolean retryable;
      private final String description;
      private final int httpStatus;

      private Reason(boolean retryable, String description, int httpStatus) {
        this.retryable = retryable;
        this.description = description;
        this.httpStatus = httpStatus;
      }

      public boolean retryable() {
        return retryable;
      }

      public String description() {
        return description;
      }

      public int httpStatus() {
        return httpStatus;
      }
    }

    private final String reason;
    private final int httpStatus;
    private final boolean retryable;

    public DatastoreRpcException(Reason reason) {
      this(reason.name(), reason.httpStatus, reason.retryable, reason.description);
    }

    public DatastoreRpcException(String reason, int httpStatus, boolean retryable, String message) {
      super(message);
      this.reason = reason;
      this.httpStatus = httpStatus;
      this.retryable = retryable;
    }

    public String reason() {
      return reason;
    }

    public int httpStatus() {
      return httpStatus;
    }

    public boolean retryable() {
      return retryable;
    }
  }

  AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException;

  BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException;

  CommitResponse commit(CommitRequest request) throws DatastoreRpcException;

  LookupResponse lookup(LookupRequest request) throws DatastoreRpcException;

  RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException;

  RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;

/**
 * Provides access to the remote Datastore service.
 */
public interface DatastoreRpc {

  public class DatastoreRpcException extends Exception {

    /**
     * The reason for the exception.
     *
     * @see Google Cloud Datastore error codes
     */
    public enum Reason {

      ABORTED(true, "Request aborted", 409),
      DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
      FAILED_PRECONDITION(false, "Invalid request", 412),
      INTERNAL(false, "Server returned an error", 500),
      INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
      PERMISSION_DENIED(false, "Unauthorized request", 403),
      RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
      UNAVAILABLE(true, "Could not reach service", 503);

      private final boolean retryable;
      private final String description;
      private final int httpStatus;

      private Reason(boolean retryable, String description, int httpStatus) {
        this.retryable = retryable;
        this.description = description;
        this.httpStatus = httpStatus;
      }

      public boolean retryable() {
        return retryable;
      }

      public String description() {
        return description;
      }

      public int httpStatus() {
        return httpStatus;
      }
    }

    private final String reason;
    private final int httpStatus;
    private final boolean retryable;

    public DatastoreRpcException(Reason reason) {
      this(reason.name(), reason.httpStatus, reason.retryable, reason.description);
    }

    public DatastoreRpcException(String reason, int httpStatus, boolean retryable, String message) {
      super(message);
      this.reason = reason;
      this.httpStatus = httpStatus;
      this.retryable = retryable;
    }

    public String reason() {
      return reason;
    }

    public int httpStatus() {
      return httpStatus;
    }

    public boolean retryable() {
      return retryable;
    }
  }

  AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException;

  BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException;

  CommitResponse commit(CommitRequest request) throws DatastoreRpcException;

  LookupResponse lookup(LookupRequest request) throws DatastoreRpcException;

  RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException;

  RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
index 936221c24156..729144e6ca26 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
@@ -1 +1 @@
-package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.datastore.DatastoreServiceOptions;

public interface DatastoreRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.datastore.DatastoreServiceOptions;

public interface DatastoreRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
index e2dc9cb6db7b..7bcabf081ef0 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -1 +1 @@
-package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceOptions;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.dataset())
            .host(options.host())
            .initializer(options.httpTransport().createRequestFactory().getInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceOptions;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.dataset())
            .host(options.host())
            .initializer(options.httpTransport().createRequestFactory().getInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
index 521b6f435711..7d119178213a 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public Bucket bucket(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public Bucket bucket(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
index ee9fe8776518..84d31d9f339c 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -1 +1 @@
-package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

public interface ServiceRpcFactory {

  S create(O options);
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

public interface ServiceRpcFactory {

  S create(O options);
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
index f6d10dda5c9b..30caacc6df4a 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
@@ -1 +1 @@
-package com.google.gcloud.com.google.gcloud.spi;

import com.google.common.collect.Iterables;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.storage.StorageServiceOptions;

import java.util.ServiceLoader;

public class ServiceRpcProvider {

  public static DatastoreRpc datastore(DatastoreServiceOptions options) {
    DatastoreRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(DatastoreRpcFactory.class), null);
    return factory == null ? new DefaultDatastoreRpc(options) : factory.create(options);
  }

  public static StorageRpc storage(StorageServiceOptions options) {
    StorageRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(StorageRpcFactory.class), null);
    return factory == null ? new DefaultStorageRpc(options) : factory.create(options);
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.common.collect.Iterables;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.storage.StorageServiceOptions;

import java.util.ServiceLoader;

public class ServiceRpcProvider {

  public static DatastoreRpc datastore(DatastoreServiceOptions options) {
    DatastoreRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(DatastoreRpcFactory.class), null);
    return factory == null ? new DefaultDatastoreRpc(options) : factory.create(options);
  }

  public static StorageRpc storage(StorageServiceOptions options) {
    StorageRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(StorageRpcFactory.class), null);
    return factory == null ? new DefaultStorageRpc(options) : factory.create(options);
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
index a895f696e16f..b6384883e350 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
index c489346c3f30..c011954b6789 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
+++ b/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
@@ -1 +1 @@
-package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BucketImpl.java b/src/main/java/com/google/gcloud/storage/BucketImpl.java
index 8509b63655e0..2a7b8b20a6c2 100644
--- a/src/main/java/com/google/gcloud/storage/BucketImpl.java
+++ b/src/main/java/com/google/gcloud/storage/BucketImpl.java
@@ -1 +1 @@
-package com.google.gcloud.storage;

import com.google.gcloud.storage.StorageObject.Key;

import java.util.Iterator;

public class BucketImpl implements Bucket {

  private final StorageServiceImpl storageService;
  private final com.google.api.services.storage.model.Bucket model;

  BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) {
    this.storageService = storageService;
    this.model = model;
  }

  @Override
  public String id() {
    return null;
  }

  @Override
  public String name() {
    return null;
  }

  @Override
  public Cors cors() {
    return null;
  }

  @Override
  public Acl acl() {
    return null;
  }

  @Override
  public Acl defaultObjectAcl() {
    return null;
  }

  @Override
  public void updateCors(Cors cors) {

  }

  @Override
  public void updateAcl(Acl acl) {

  }

  @Override
  public void updateDefaultObjectAcl() {

  }

  @Override
  public void delete(String... objectName) {

  }

  @Override
  public void compose(Iterable sourceObjectNames, String destObjectName) {

  }

  @Override
  public void copy(String sourceObjectName, Key destObjectKey) {

  }

  @Override
  public StorageObject get(String objectName) {
    return null;
  }

  @Override
  public Iterator get(String... objectName) {
    return null;
  }

  @Override
  public InputChannel getInputChannel(String ObjectName) {
    return null;
  }

  @Override
  public OutputChannel getOutputChannel(String ObjectName) {
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import com.google.gcloud.storage.StorageObject.Key;

import java.util.Iterator;

public class BucketImpl implements Bucket {

  private final StorageServiceImpl storageService;
  private final com.google.api.services.storage.model.Bucket model;

  BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) {
    this.storageService = storageService;
    this.model = model;
  }

  @Override
  public String id() {
    return null;
  }

  @Override
  public String name() {
    return null;
  }

  @Override
  public Cors cors() {
    return null;
  }

  @Override
  public Acl acl() {
    return null;
  }

  @Override
  public Acl defaultObjectAcl() {
    return null;
  }

  @Override
  public void updateCors(Cors cors) {

  }

  @Override
  public void updateAcl(Acl acl) {

  }

  @Override
  public void updateDefaultObjectAcl() {

  }

  @Override
  public void delete(String... objectName) {

  }

  @Override
  public void compose(Iterable sourceObjectNames, String destObjectName) {

  }

  @Override
  public void copy(String sourceObjectName, Key destObjectKey) {

  }

  @Override
  public StorageObject get(String objectName) {
    return null;
  }

  @Override
  public Iterator get(String... objectName) {
    return null;
  }

  @Override
  public InputChannel getInputChannel(String ObjectName) {
    return null;
  }

  @Override
  public OutputChannel getOutputChannel(String ObjectName) {
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
index 9779b20aabcf..8d181b182637 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceException.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
@@ -1 +1 @@
-package com.google.gcloud.storage;

import java.io.IOException;

public class StorageServiceException extends RuntimeException {

  StorageServiceException(IOException ex) {
    super(ex);
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.IOException;

public class StorageServiceException extends RuntimeException {

  StorageServiceException(IOException ex) {
    super(ex);
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
index 4b2404358082..8b164455c001 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
@@ -1 +1 @@
-package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retriable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retriable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
index d54d87814b60..4614b75c9e06 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static junit.framework.TestCase.assertEquals;
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index eb2d1567cb29..c58253b3189e 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static junit.framework.TestCase.assertEquals;
diff --git a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
index 601f73659fbd..cab1fef6ca84 100644
--- a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.gcloud.datastore;
 
 import static junit.framework.TestCase.assertEquals;

From 6935727fdfa421765efc9c95f99e4cfac816cc6b Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Mon, 2 Mar 2015 08:24:21 -0800
Subject: [PATCH 123/732] storage work in progress

---
 .../java/com/google/gcloud/storage/Acl.java   |  48 +++-
 .../com/google/gcloud/storage/Bucket.java     |  40 +--
 .../com/google/gcloud/storage/BucketImpl.java |   2 +-
 .../com/google/gcloud/storage/BucketInfo.java | 108 +++++++
 .../java/com/google/gcloud/storage/Cors.java  |  25 +-
 .../com/google/gcloud/storage/ObjectInfo.java | 266 ++++++++++++++++++
 ...putChannel.java => ObjectReadChannel.java} |   8 +-
 ...utChannel.java => ObjectWriteChannel.java} |   8 +-
 .../google/gcloud/storage/StorageObject.java  |  94 -------
 .../google/gcloud/storage/StorageService.java |  35 ++-
 10 files changed, 480 insertions(+), 154 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/BucketInfo.java
 create mode 100644 src/main/java/com/google/gcloud/storage/ObjectInfo.java
 rename src/main/java/com/google/gcloud/storage/{InputChannel.java => ObjectReadChannel.java} (83%)
 rename src/main/java/com/google/gcloud/storage/{OutputChannel.java => ObjectWriteChannel.java} (83%)
 delete mode 100644 src/main/java/com/google/gcloud/storage/StorageObject.java

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 4c218848ea9a..aed45c58ffe5 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -16,7 +16,22 @@
 
 package com.google.gcloud.storage;
 
-public interface Acl {
+public final class Acl {
+
+  private final String domain;
+  private final Entity entity;
+  private final String entityId;
+
+  String email();
+
+  String etag();
+
+  String generation();
+
+
+  ProjectTeam projectTeam();
+
+  String role();
 
   class ProjectTeam {
       // ProjectNumber: The project number.
@@ -26,18 +41,29 @@ class ProjectTeam {
   //Team string `json:"team,omitempty"`
   }
 
-  enum Entity {
-    USER_ID("user-userId"),
-    USER_EMAIL("user-emailAddress"),
-    GROUP_ID("group-groupId"),
-    GROUP_EMAIL("group-emailAddress"),
-    ALL_USERS("allUsers"),
-    ALL_AUTHENTICATED_USERS("allAuthenticatedUsers");
+  public static abstract class Entity {
+
+    private final Type type;
 
-    private final String value;
+    public static final Entity ALL_USERS = new Entity(Type.ALL_USERS);
+
+    public enum Type {
+      USER_ID("user-userId"),
+      USER_EMAIL("user-emailAddress"),
+      GROUP_ID("group-groupId"),
+      GROUP_EMAIL("group-emailAddress"),
+      ALL_USERS("allUsers"),
+      ALL_AUTHENTICATED_USERS("allAuthenticatedUsers");
+
+      private final String value;
+
+      Type(String value) {
+        this.value = value;
+      }
+    }
 
-    Entity(String value) {
-      this.value = value;
+    Entity(EntityType type) {
+      this.type = type;
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 86942b1779e2..13e615e8e1b5 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -16,47 +16,9 @@
 
 package com.google.gcloud.storage;
 
-import java.util.Iterator;
-
 public interface Bucket {
 
-  String id();
-
-  String name();
-
-  Cors cors();
-
-  Acl acl();
-
-  Acl defaultObjectAcl();
-
-  void updateCors(Cors cors);
-
-  void updateAcl(Acl acl);
-
-  void updateDefaultObjectAcl();
-
-  void delete(String... objectName);
-
-  void compose(Iterable sourceObjectNames, String destObjectName);
-
-  void copy(String sourceObjectName, StorageObject.Key destObjectKey);
-
-
-  // TODO (ozarov): consider replace with Object that has a reference to bucket and name
-  // that object can return its own meta-data, update its own meta-data, replace its content
-  // via a stream or byteBuffer, read its content (via stream or ByteBuffer),...
-  //void copy(String source, String bucket, String dest);
-  // Also consider read with an offset (and limit).
-
-  // returns null if not exists
-  StorageObject get(String objectName);
-
-  Iterator get(String... objectName);
-
-  InputChannel getInputChannel(String ObjectName);
 
-  OutputChannel getOutputChannel(String ObjectName);
+  BucketInfo info();
 
-  // TODO: add listing
 }
diff --git a/src/main/java/com/google/gcloud/storage/BucketImpl.java b/src/main/java/com/google/gcloud/storage/BucketImpl.java
index 2a7b8b20a6c2..addf35b8f8de 100644
--- a/src/main/java/com/google/gcloud/storage/BucketImpl.java
+++ b/src/main/java/com/google/gcloud/storage/BucketImpl.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import com.google.gcloud.storage.StorageObject.Key;

import java.util.Iterator;

public class BucketImpl implements Bucket {

  private final StorageServiceImpl storageService;
  private final com.google.api.services.storage.model.Bucket model;

  BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) {
    this.storageService = storageService;
    this.model = model;
  }

  @Override
  public String id() {
    return null;
  }

  @Override
  public String name() {
    return null;
  }

  @Override
  public Cors cors() {
    return null;
  }

  @Override
  public Acl acl() {
    return null;
  }

  @Override
  public Acl defaultObjectAcl() {
    return null;
  }

  @Override
  public void updateCors(Cors cors) {

  }

  @Override
  public void updateAcl(Acl acl) {

  }

  @Override
  public void updateDefaultObjectAcl() {

  }

  @Override
  public void delete(String... objectName) {

  }

  @Override
  public void compose(Iterable sourceObjectNames, String destObjectName) {

  }

  @Override
  public void copy(String sourceObjectName, Key destObjectKey) {

  }

  @Override
  public StorageObject get(String objectName) {
    return null;
  }

  @Override
  public Iterator get(String... objectName) {
    return null;
  }

  @Override
  public InputChannel getInputChannel(String ObjectName) {
    return null;
  }

  @Override
  public OutputChannel getOutputChannel(String ObjectName) {
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import com.google.gcloud.storage.StorageObject.Key;

import java.util.Iterator;

public class BucketImpl implements Bucket {

  private final StorageServiceImpl storageService;
  private final com.google.api.services.storage.model.Bucket model;

  BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) {
    this.storageService = storageService;
    this.model = model;
  }

  @Override
  public String id() {
    return null;
  }

  @Override
  public String name() {
    return null;
  }

  @Override
  public Cors cors() {
    return null;
  }

  @Override
  public Acl acl() {
    return null;
  }

  @Override
  public Acl defaultObjectAcl() {
    return null;
  }

  @Override
  public void updateCors(Cors cors) {

  }

  @Override
  public void updateAcl(Acl acl) {

  }

  @Override
  public void updateDefaultObjectAcl() {

  }

  @Override
  public void delete(String... objectName) {

  }

  @Override
  public void compose(Iterable sourceObjectNames, String destObjectName) {

  }

  @Override
  public void copy(String sourceObjectName, Key destObjectKey) {

  }

  @Override
  public StorageObject get(String objectName) {
    return null;
  }

  @Override
  public Iterator get(String... objectName) {
    return null;
  }

  @Override
  public ObjectReadChannel getInputChannel(String ObjectName) {
    return null;
  }

  @Override
  public ObjectWriteChannel getOutputChannel(String ObjectName) {
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
new file mode 100644
index 000000000000..2bf3900002c3
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+public final class BucketInfo {
+
+  private final String id;
+  private final String name;
+  private final long createTime;
+  private final long metageneration;
+  private final Cors cors;
+  private final Acl acl;
+  private final Acl defaultAcl;
+
+  public enum StorageClass {
+    DURABLE_REDUCED_AVAILABILITY,
+    STANDARD
+  }
+
+  public final static class Builder {
+
+    private final String id;
+    private final String name;
+    private final long createTime;
+    private final long metageneration;
+    private Cors cors;
+    private Acl acl;
+    private Acl defaultAcl;
+
+    Builder(String id, String name, long createTime, long metageneration) {
+      this.id = id;
+      this.name = name;
+      this.createTime = createTime;
+      this.metageneration = metageneration;
+    }
+
+    public Builder cors(Cors cors) {
+      this.cors = cors;
+      return this;
+    }
+
+    public Builder acl(Acl acl) {
+      this.acl = acl;
+      return this;
+    }
+
+    public Builder defaultAcl(Acl defaultAcl) {
+      this.defaultAcl = defaultAcl;
+      return this;
+    }
+
+    public BucketInfo build() {
+      return new BucketInfo(this);
+    }
+  }
+
+  private BucketInfo(Builder builder) {
+    id = builder.id;
+    name = builder.name;
+    createTime = builder.createTime;
+    metageneration = builder.metageneration;
+    cors = builder.cors;
+    acl = builder.acl;
+    defaultAcl = builder.defaultAcl;
+  }
+
+  public String id() {
+    return id;
+  }
+
+  public String name() {
+    return name;
+  }
+
+  public Cors cors() {
+    return cors;
+  }
+
+  public Acl acl() {
+    return acl;
+  }
+
+  public Acl defaultObjectAcl() {
+    return defaultAcl;
+  }
+
+  public Builder toBuilder() {
+    return builder(id, name, createTime, metageneration).cors(cors).acl(acl).defaultAcl(defaultAcl);
+  }
+
+  public static Builder builder(String id, String name, long createTime, long metageneration) {
+    return new Builder(id, name, createTime, metageneration);
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index 71e87da50fd3..c5a944cd6ac9 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -22,7 +22,7 @@
 import java.net.URISyntaxException;
 import java.util.List;
 
-public class Cors {
+public final class Cors {
 
   private final Integer maxAgeSeconds;
   private final ImmutableList methods;
@@ -73,16 +73,31 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) {
       return this;
     }
 
+    public Builder methods(List methods) {
+      this.methods = ImmutableList.copyOf(methods);
+      return this;
+    }
+
     public Builder methods(Method... methods) {
       this.methods = ImmutableList.copyOf(methods);
       return this;
     }
 
+    public Builder origins(List origins) {
+      this.origins = ImmutableList.copyOf(origins);
+      return this;
+    }
+
     public Builder origins(Origin... origins) {
       this.origins = ImmutableList.copyOf(origins);
       return this;
     }
 
+    public Builder responseHeaders(List headers) {
+      this.responseHeaders = ImmutableList.copyOf(headers);
+      return this;
+    }
+
     public Builder responseHeaders(String... responseHeaders) {
       this.responseHeaders = ImmutableList.copyOf(responseHeaders);
       return this;
@@ -116,6 +131,14 @@ public List responseHeaders() {
     return responseHeaders;
   }
 
+  public Builder toBuilder() {
+    return builder()
+        .maxAgeSeconds(maxAgeSeconds)
+        .methods(methods)
+        .origins(origins)
+        .responseHeaders(responseHeaders);
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
new file mode 100644
index 000000000000..97183bc720ea
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
@@ -0,0 +1,266 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+
+import java.util.List;
+import java.util.Map;
+
+public class ObjectInfo {
+
+  private final String bucket;
+  private final String name;
+  private final String contentType;
+  private final String cacheControl;
+  private final ImmutableList acl;
+  private final String owner;
+  private final long size;
+  private final String contentEncoding;
+  private final HashCode md5;
+  private final long crc32;
+  private final String mediaLink;
+  private final ImmutableMap metadata;
+  private final long generation;
+  private final long metageneration;
+  private final long deleteTime;
+  private final long updateTime;
+
+
+  public static final class Builder {
+
+    private String bucket;
+    private String name;
+    private String contentType;
+    private String cacheControl;
+    private ImmutableList acl;
+    private String owner;
+    private long size;
+    private String contentEncoding;
+    private HashCode md5;
+    private long crc32;
+    private String mediaLink;
+    private ImmutableMap metadata;
+    private long generation;
+    private long metageneration;
+    private long deleteTime;
+    private long updateTime;
+
+    private Builder() {
+    }
+
+    public Builder bucket(String bucket) {
+      this.bucket = bucket;
+      return this;
+    }
+
+    public Builder name(String name) {
+      this.name = name;
+      return this;
+    }
+
+    public Builder contentType(String contentType) {
+      this.contentType = contentType;
+      return this;
+    }
+
+    public Builder cacheControl(String cacheControl) {
+      this.cacheControl = cacheControl;
+      return this;
+    }
+
+    public Builder acl(List acl) {
+      this.acl = ImmutableList.copyOf(acl);
+      return this;
+    }
+
+    public Builder owner(String owner) {
+      this.owner = owner;
+      return this;
+    }
+
+    public Builder size(long size) {
+      this.size = size;
+      return this;
+    }
+
+    public Builder contentEncoding(String contentEncoding) {
+      this.contentEncoding = contentEncoding;
+      return this;
+    }
+
+    Builder md5(HashCode md5) {
+      this.md5 = md5;
+      return this;
+    }
+
+    public Builder md5(byte[] bytes) {
+      this.md5 = bytes == null ? null : Hashing.md5().hashBytes(bytes);
+      return this;
+    }
+
+    public Builder crc32(long crc32) {
+      this.crc32 = crc32;
+      return this;
+    }
+
+    public Builder mediaLink(String mediaLink) {
+      this.mediaLink = mediaLink;
+      return this;
+    }
+
+    public Builder metadata(Map metadata) {
+      this.metadata = ImmutableMap.copyOf(metadata);
+      return this;
+    }
+
+    public Builder generation(long generation) {
+      this.generation = generation;
+      return this;
+    }
+
+    public Builder metageneration(long metageneration) {
+      this.metageneration = metageneration;
+      return this;
+    }
+
+    public Builder deleteTime(long deleteTime) {
+      this.deleteTime = deleteTime;
+      return this;
+    }
+
+    public Builder updateTime(long updateTime) {
+      this.updateTime = updateTime;
+      return this;
+    }
+
+    public ObjectInfo build() {
+      return new ObjectInfo(this);
+    }
+  }
+
+  private ObjectInfo(Builder builder) {
+    bucket = builder.bucket;
+    name = builder.name;
+    contentType = builder.contentType;
+    cacheControl = builder.cacheControl;
+    acl = builder.acl;
+    owner = builder.owner;
+    size = builder.size;
+    contentEncoding = builder.contentEncoding;
+    md5 = builder.md5;
+    crc32 = builder.crc32;
+    mediaLink = builder.mediaLink;
+    metadata = builder.metadata;
+    generation = builder.generation;
+    metageneration = builder.metageneration;
+    deleteTime = builder.deleteTime;
+    updateTime = builder.updateTime;
+  }
+
+  public String bucket() {
+    return bucket;
+  }
+
+  public String name() {
+    return name;
+  }
+
+  public String contentType() {
+    return contentType;
+  }
+
+  public String cacheControl() {
+    return cacheControl;
+  }
+
+  public List acl() {
+    return acl;
+  }
+
+  public String owner() {
+    return owner;
+  }
+
+  public long size() {
+    return size;
+  }
+
+  public String contentEncoding() {
+    return contentEncoding;
+  }
+
+  public byte[] md5() {
+    return md5 == null ? null : md5.asBytes();
+  }
+
+  public long crc32() {
+    return crc32;
+  }
+
+  public String mediaLink() {
+    return mediaLink;
+  }
+
+  public Map metadata() {
+    return metadata;
+  }
+
+  public long generation() {
+    return generation;
+  }
+
+  public long metageneration() {
+    return metageneration;
+  }
+
+  public long deleteTime() {
+    return deleteTime;
+  }
+
+  public long updateTime() {
+    return updateTime;
+  }
+
+  public Builder toBuilder() {
+    return builder()
+        .acl(acl)
+        .bucket(bucket)
+        .cacheControl(cacheControl)
+        .contentEncoding(contentEncoding)
+        .crc32(crc32)
+        .contentType(contentType)
+        .deleteTime(deleteTime)
+        .generation(generation)
+        .md5(md5)
+        .mediaLink(mediaLink)
+        .metadata(metadata)
+        .metageneration(metageneration)
+        .name(name)
+        .owner(owner)
+        .updateTime(updateTime)
+        .size(size);
+
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/InputChannel.java b/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java
similarity index 83%
rename from src/main/java/com/google/gcloud/storage/InputChannel.java
rename to src/main/java/com/google/gcloud/storage/ObjectReadChannel.java
index 901ccf048ee9..0dbb320685f1 100644
--- a/src/main/java/com/google/gcloud/storage/InputChannel.java
+++ b/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java
@@ -21,15 +21,17 @@
 import java.nio.channels.ReadableByteChannel;
 
 /**
- * A readable byte channel for reading data from Google Cloud Storage.
+ * A channel for reading data from a Google Cloud Storage object.
  *
  *  Implementations of this class may buffer data internally to reduce remote calls.
  *
  *  This class is @{link Serializable}, which allows incremental reads.
  */
-public interface InputChannel extends ReadableByteChannel, Serializable, Closeable {
+public interface ObjectReadChannel extends ReadableByteChannel, Serializable, Closeable {
 
-  StorageObject.Key key();
+  String bucket();
+
+  String object();
 
   int size();
 
diff --git a/src/main/java/com/google/gcloud/storage/OutputChannel.java b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
similarity index 83%
rename from src/main/java/com/google/gcloud/storage/OutputChannel.java
rename to src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
index cf45e292ce2c..8cc86d2c3e00 100644
--- a/src/main/java/com/google/gcloud/storage/OutputChannel.java
+++ b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
@@ -21,13 +21,15 @@
 import java.nio.channels.WritableByteChannel;
 
 /**
- * A writable byte channel for writing data to Google Cloud Storage.
+ * A channel for writing data to a Google Cloud Storage object.
  *
  * Implementations of this class may further buffer data internally to reduce remote calls.
  * Written data will only be visible after calling {@link #close()}.
  * This class is serializable, to allow incremental writes.
  */
-public interface OutputChannel extends WritableByteChannel, Serializable, Closeable {
+public interface ObjectWriteChannel extends WritableByteChannel, Serializable, Closeable {
 
-  StorageObject.Key key();
+  String bucket();
+
+  String object();
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageObject.java b/src/main/java/com/google/gcloud/storage/StorageObject.java
deleted file mode 100644
index 2a8dcac2f916..000000000000
--- a/src/main/java/com/google/gcloud/storage/StorageObject.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.storage;
-
-import java.nio.ByteBuffer;
-
-// TODO: add  equals,hashCode, toString, serializable
-public interface StorageObject {
-
-  class Key {
-
-    // TODO: add builder, factory method, toURL, from URL, equals,hashCode, toString, serializable
-    private final String bucket;
-    private final String name;
-
-
-    Key(Bucket bucket, String name) {
-      this.bucket = bucket.name();
-      this.name = name;
-    }
-
-    public String bucket() {
-      return bucket;
-    }
-
-    public String name() {
-      return name;
-    }
-  }
-
-  abstract class Builder {
-
-    private Bucket bucket;
-    private Acl acl;
-    private String name;
-    private ByteBuffer content;
-
-    public Builder() {
-
-    }
-
-    public Builder name(String name) {
-      this.name = name;
-      return this;
-    }
-
-    public Builder bucket(Bucket bucket) {
-      this.bucket = bucket;
-      return this;
-    }
-
-    public Builder acl(Acl acl) {
-      this.acl = acl;
-      return this;
-    }
-
-    public Builder content(ByteBuffer content) {
-      this.content = content;
-      return this;
-    }
-
-    public abstract StorageObject build();
-  }
-
-  boolean exists();
-
-  Key key();
-
-  Acl acl();
-
-  ByteBuffer content();
-
-  void save();
-
-  void delete();
-
-  InputChannel getInputChannel();
-
-  OutputChannel getOutputChannel();
-}
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 3fd0b6b3fa1b..77147b3621ea 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -18,9 +18,40 @@
 
 import com.google.gcloud.Service;
 
+import java.util.Iterator;
+
 public interface StorageService extends Service {
 
-  Iterable listBuckets();
+  Iterable list();
+
+  Iterable list(ListSettings settings);
+
+  BucketInfo get(String bucket);
+
+  Iterator get(String... bucket);
+
+  ObjectInfo get(String bucket, String object);
+
+  Iterator get(String bucket, String... object);
+
+  void update(BucketInfo bucketInfo);
+
+  void update(ObjectInfo objectInfo);
+
+  void delete(String bucket, String... object);
+
+  void compose(String bucket, Iterable sourceObjects, String destObject);
+
+  void copy(String fromBucket, String fromObject, String destBucket, String destObject);
+
+
+  ObjectReadChannel getInputChannel(String bucket, String ObjectName);
+
+  ObjectWriteChannel write(String ObjectName);
+
+  // TODO: add listing
+
+  ObjectReadChannel getInputChannel();
 
-  Bucket getBucket(String bucketName);
+  ObjectWriteChannel getOutputChannel();
 }

From fe435d1ddef53b3029a43b96601ebd3621da9ffb Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 2 Mar 2015 17:36:08 -0800
Subject: [PATCH 124/732] wip

---
 .../java/com/google/gcloud/storage/Acl.java   | 91 +++++++++----------
 1 file changed, 44 insertions(+), 47 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index aed45c58ffe5..280139035745 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -16,71 +16,68 @@
 
 package com.google.gcloud.storage;
 
-public final class Acl {
+import java.io.Serializable;
 
-  private final String domain;
-  private final Entity entity;
-  private final String entityId;
+public abstract class Acl implements Serializable {
 
-  String email();
+  private static final long serialVersionUID = 6435575339887912222L;
 
-  String etag();
+  private final Type type;
+  private final Role role;
 
-  String generation();
-
-
-  ProjectTeam projectTeam();
-
-  String role();
-
-  class ProjectTeam {
-      // ProjectNumber: The project number.
-  //ProjectNumber string `json:"projectNumber,omitempty"`
-
-  // Team: The team. Can be owners, editors, or viewers.
-  //Team string `json:"team,omitempty"`
+  public enum Role {
+    OWNER,
+    READER,
+    WRITER
   }
 
-  public static abstract class Entity {
-
-    private final Type type;
+  public enum Type {
+    DOMAIN,
+    GROUP,
+    USER,
+    PROJECT
+  }
 
-    public static final Entity ALL_USERS = new Entity(Type.ALL_USERS);
+  public static class User extends Acl {
 
-    public enum Type {
-      USER_ID("user-userId"),
-      USER_EMAIL("user-emailAddress"),
-      GROUP_ID("group-groupId"),
-      GROUP_EMAIL("group-emailAddress"),
-      ALL_USERS("allUsers"),
-      ALL_AUTHENTICATED_USERS("allAuthenticatedUsers");
+    private static final long serialVersionUID = 3076518036392737008L;
 
-      private final String value;
+    private String email;
 
-      Type(String value) {
-        this.value = value;
-      }
+    User(Role role, String email) {
+      super(Type.USER, role);
+      this.email = email;
     }
 
-    Entity(EntityType type) {
-      this.type = type;
+    String email() {
+      return email;
     }
-  }
-
-  String domain();
 
-  Entity entity();
-
-  String entityId();
+    public static User forEmail(Role role, String email) {
+      return new User(role, email);
+    }
 
-  String email();
+    public static User allUsers(Role role) {
+      return forEmail(role, "allUsers");
+    }
 
-  String etag();
+    public static User allAuthenticatedUsers(Role role) {
+      return forEmail(role, "allAuthenticatedUsers");
+    }
+  }
 
-  String generation();
+  
+  Acl(Type type, Role role) {
+    this.type = type;
+    this.role = role;
+  }
 
 
-  ProjectTeam projectTeam();
+  public Type type() {
+    return type;
+  }
 
-  String role();
+  public Role role() {
+    return role;
+  }
 }

From 01ba0996a6e01981c35713ec952d67fbb1b79307 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Tue, 3 Mar 2015 13:24:28 -0800
Subject: [PATCH 125/732] Update Query Javadoc regarding projection type

---
 src/main/java/com/google/gcloud/datastore/Query.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index a8f4c97fdc72..2dd3b231e6ee 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -45,7 +45,7 @@ public abstract class Query extends Serializable {
   /**
    * This class represents the expected type of the result.
    *   FULL: A complete {@link Entity}.
-   *   PROJECTION: A partial entity, represented by {@link PartialEntity}.
+   *   PROJECTION: A projection entity, represented by {@link ProjectionEntity}.
    *   KEY_ONLY: An entity's {@link Key}.
    */
   public abstract static class Type implements java.io.Serializable {

From a0fd66c4e77330f71a3ff0721d3cddc4c4d11d49 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 3 Mar 2015 17:15:31 -0800
Subject: [PATCH 126/732] Make DatastoreHelper internal

---
 .../gcloud/datastore/DatastoreHelper.java     | 108 ++----------------
 .../gcloud/datastore/DatastoreReader.java     |   8 ++
 .../gcloud/datastore/DatastoreService.java    |  21 ++++
 .../datastore/DatastoreServiceImpl.java       |  14 +++
 .../google/gcloud/datastore/QueryResult.java  |   3 +-
 .../google/gcloud/datastore/Transaction.java  |  10 ++
 .../gcloud/datastore/TransactionImpl.java     |   6 +
 .../gcloud/datastore/DatastoreHelperTest.java |  94 +--------------
 .../datastore/DatastoreServiceTest.java       |  32 +++---
 9 files changed, 87 insertions(+), 209 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index 6b89b9f3ab2a..bca038e88b35 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -24,100 +24,20 @@
 import java.util.Map;
 
 /**
- * Adds some functionality to DatastoreService that should
- * be provided statically to the interface (Java 8).
+ * Provide functionality that should be added to the appropriate interfaces
+ * via Java 8 default methods.
  */
-public class DatastoreHelper implements DatastoreService {
+class DatastoreHelper {
 
-  private final DatastoreService delegate;
-
-  private DatastoreHelper(DatastoreService delegate) {
-    this.delegate = delegate;
-  }
-
-  @Override
-  public Entity get(Key key) {
-    return delegate.get(key);
-  }
-
-  @Override
-  public Iterator get(Key... key) {
-    return delegate.get(key);
-  }
-
-  @Override
-  public  QueryResult run(Query query) {
-    return delegate.run(query);
-  }
-
-  @Override
-  public DatastoreServiceOptions options() {
-    return delegate.options();
-  }
-
-  @Override
-  public Transaction newTransaction(TransactionOption... options) {
-    return delegate.newTransaction(options);
-  }
-
-  @Override
-  public Batch newBatch(BatchOption... options) {
-    return delegate.newBatch(options);
-  }
-
-  @Override
-  public Key allocateId(PartialKey key) {
-    return delegate.allocateId(key);
-  }
-
-  @Override
-  public List allocateId(PartialKey... key) {
-    return delegate.allocateId(key);
-  }
-
-  @Override
-  public Entity add(PartialEntity entity) {
-    return delegate.add(entity);
-  }
-
-  @Override
-  public List add(PartialEntity... entity) {
-    return delegate.add(entity);
-  }
-
-  @Override
-  public void add(Entity... entity) {
-    delegate.add(entity);
-  }
-
-  @Override
-  public void update(Entity... entity) {
-    delegate.update(entity);
-  }
-
-  @Override
-  public void put(Entity... entity) {
-    delegate.put(entity);
-  }
-
-  @Override
-  public void delete(Key... key) {
-    delegate.delete(key);
-  }
-
-  /**
-   * Returns a new KeyFactory for this service
-   */
-  public KeyFactory newKeyFactory() {
-    return new KeyFactory(delegate);
+  private DatastoreHelper() {
   }
 
   /**
    * Returns a list with a value for each given key (ordered by input).
    * A {@code null} would be returned for non-existing keys.
    */
-  public List fetch(Key... keys) {
-    Iterator entities = delegate.get(keys);
+  static List fetch(DatastoreReader reader, Key... keys) {
+    Iterator entities = reader.get(keys);
     Map map = Maps.newHashMapWithExpectedSize(keys.length);
     while (entities.hasNext()) {
       Entity entity = entities.next();
@@ -131,12 +51,9 @@ public List fetch(Key... keys) {
     return list;
   }
 
-  public interface RunInTransaction {
-    void run(DatastoreReaderWriter readerWriter);
-  }
-
-  public void runInTransaction(RunInTransaction runFor, TransactionOption... options) {
-    Transaction transaction = newTransaction(options);
+  static void runInTransaction(DatastoreService datastoreService,
+      DatastoreService.RunInTransaction runFor, TransactionOption... options) {
+    Transaction transaction = datastoreService.newTransaction(options);
     try {
       runFor.run(transaction);
       transaction.commit();
@@ -146,11 +63,4 @@ public void runInTransaction(RunInTransaction runFor, TransactionOption... optio
       }
     }
   }
-
-  public static DatastoreHelper createFor(DatastoreService datastoreService) {
-    if (datastoreService instanceof DatastoreHelper) {
-      return (DatastoreHelper) datastoreService;
-    }
-    return new DatastoreHelper(datastoreService);
-  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index 5c9ceb19cc35..7b24dbba3f25 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -17,6 +17,7 @@
 package com.google.gcloud.datastore;
 
 import java.util.Iterator;
+import java.util.List;
 
 /**
  * An interface to represent Google Cloud Datastore read operations.
@@ -42,6 +43,13 @@ public interface DatastoreReader {
    */
   Iterator get(Key... key);
 
+  /**
+   * Returns a list with a value for each given key (ordered by input).
+   * A {@code null} would be returned for non-existing keys.
+   * When possible prefer using {@link #get(Key...)} which does not eagerly loads the results.
+   */
+  List fetch(Key... keys);
+
   /**
    * Submit a {@link Query} and returns its result.
    *
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index 7d78192a5a1f..395ceb2fc873 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -32,6 +32,22 @@ public interface DatastoreService extends Service, Data
    */
   Transaction newTransaction(TransactionOption... options);
 
+  interface RunInTransaction {
+    void run(DatastoreReaderWriter readerWriter);
+  }
+
+
+  /**
+   * Invokes the callback's {@link RunInTransaction#run} method with a
+   * {@link DatastoreReaderWriter} that is associated with a new transaction.
+   * The transaction will be committed upon successful invocation or rollback
+   * otherwise.
+   *
+   * @param runFor the functor to call with the transactional readerWriter
+   * @param options the options for the created transaction
+   */
+  void runInTransaction(RunInTransaction runFor, TransactionOption... options);
+
   /**
    * Returns a new Batch for processing multiple write operations in one request.
    */
@@ -105,4 +121,9 @@ public interface DatastoreService extends Service, Data
    */
   @Override
   void delete(Key... key);
+
+  /**
+   * Returns a new KeyFactory for this service
+   */
+  KeyFactory newKeyFactory();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 231624445e76..52430a61c86d 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -90,6 +90,10 @@ public Transaction newTransaction(TransactionOption... options) {
     return new TransactionImpl(this, options);
   }
 
+  public void runInTransaction(RunInTransaction runFor, TransactionOption... options) {
+    DatastoreHelper.runInTransaction(this, runFor, options);
+  }
+
   @Override
   public  QueryResult run(Query query) {
     return run(null, query);
@@ -213,6 +217,11 @@ public Iterator get(Key... keys) {
     return get(null, keys);
   }
 
+  @Override
+  public List fetch(Key... keys) {
+    return DatastoreHelper.fetch(this, keys);
+  }
+
   Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) {
     if (keys.length == 0) {
       return Collections.emptyIterator();
@@ -331,6 +340,11 @@ public void delete(Key... keys) {
     }
   }
 
+  @Override
+  public KeyFactory newKeyFactory() {
+    return new KeyFactory(this);
+  }
+
   private DatastoreV1.CommitResponse commitMutation(DatastoreV1.Mutation.Builder mutationPb) {
     if (options().force()) {
       mutationPb.setForce(true);
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResult.java
index 65d087d4b4a9..45cca8c13dd3 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResult.java
@@ -35,7 +35,8 @@ public interface QueryResult extends Iterator {
   Class resultClass();
 
   /**
-   * Returns the Cursor for the next result. Not currently implemented (depends on v1beta3).
+   * Returns the Cursor for the {@link #next} result.
+   * Not currently implemented (depends on v1beta3).
    */
   Cursor cursor();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index 28592552787b..c749a007f800 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -78,6 +78,16 @@ interface Response {
   @Override
   Iterator get(Key... key);
 
+  /**
+   * {@inheritDoc}
+   * The requested entities will be part of this Datastore transaction (so a commit is guaranteed
+   * to fail if any of the entities was changed by others after they were seen by this transaction)
+   * but any write changes in this transaction will not be reflected by the returned entities.
+   *
+   * @throws DatastoreServiceException upon failure or if no longer active
+   */
+  List fetch(Key... keys);
+
   /**
    * {@inheritDoc}
    * The entities returned by the result of this query will be part of this Datastore transaction
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 859f637d4c72..2118153c252d 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -83,6 +83,12 @@ public Iterator get(Key... keys) {
     return datastore.get(readOptionsPb.build(), keys);
   }
 
+  @Override
+  public List fetch(Key... keys) {
+    validateActive();
+    return DatastoreHelper.fetch(this, keys);
+  }
+
   @Override
   public  QueryResult run(Query query) {
     validateActive();
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 915d792a82fa..580114780a90 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -17,109 +17,21 @@
 package com.google.gcloud.datastore;
 
 import static junit.framework.TestCase.assertTrue;
-import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.createStrictMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
 
 import com.google.common.collect.Iterators;
-import com.google.common.collect.Sets;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
 
 import org.junit.Test;
 
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
 public class DatastoreHelperTest {
 
-  @Test
-  public void testDelegate() throws Exception {
-    final Set methods = Sets.newHashSet(DatastoreService.class.getMethods());
-    final Map params = new HashMap<>();
-    InvocationHandler handler = new InvocationHandler() {
-      @Override
-      public Object invoke(Object o, Method method, Object[] objects) throws Throwable {
-        String methodName = Thread.currentThread().getStackTrace()[3].getMethodName();
-        if (method.getParameterTypes().length > 0) {
-          Class paramClass = method.getParameterTypes()[0];
-          methodName += paramClass.getSimpleName();
-          if (method.isVarArgs()) {
-            objects = (Object[]) objects[0];
-          }
-          assertArrayEquals(objects, params.get(methodName));
-        }
-        assertTrue(methods.remove(method));
-        return null;
-      }
-    };
-    PartialKey pKey1 = PartialKey.builder("ds", "k").build();
-    PartialKey pKey2 = PartialKey.builder("ds", "k").build();
-    PartialEntity pEntity1 = PartialEntity.builder(pKey1).build();
-    PartialEntity pEntity2 = PartialEntity.builder(pKey2).build();
-    Key key1 = Key.builder(pKey1, 1).build();
-    Key key2 = Key.builder(pKey1, "a").build();
-    Entity entity1 = Entity.builder(key1).build();
-    Entity entity2 = Entity.builder(key2).build();
-    ClassLoader cl = DatastoreService.class.getClassLoader();
-    Class[] interfaces = DatastoreHelper.class.getInterfaces();
-    DatastoreService delegate = (DatastoreService) Proxy.newProxyInstance(cl, interfaces, handler);
-    DatastoreHelper helper = DatastoreHelper.createFor(delegate);
-    params.put("getKey", new Object[] {key1});
-    helper.get(key1);
-    params.put("getKey[]", new Object[] {key1, key2});
-    helper.get(key1, key2);
-    params.put("addEntity[]", new Object[] {entity1, entity2});
-    helper.add(entity1, entity2);
-    params.put("updateEntity[]", new Object[] {entity1});
-    helper.update(entity1);
-    params.put("addPartialEntity", new Object[] {pEntity1});
-    helper.add(pEntity1);
-    params.put("addPartialEntity[]", new Object[] {pEntity2, entity1});
-    helper.add(pEntity2, entity1);
-    params.put("allocateIdPartialKey", new Object[] {pKey1});
-    helper.allocateId(pKey1);
-    params.put("allocateIdPartialKey[]", new Object[] {pKey1, pKey2});
-    helper.allocateId(pKey1, pKey2);
-    params.put("deleteKey[]", new Object[] {key1});
-    helper.delete(key1);
-    params.put("putEntity[]", new Object[] {entity1});
-    helper.put(entity1);
-    helper.options();
-    params.put("newBatchBatchOption[]", new Object[] {});
-    helper.newBatch();
-    params.put("newTransactionTransactionOption[]", new Object[] {});
-    helper.newTransaction();
-    Query query = createMock(Query.class);
-    params.put("runQuery", new Object[] {query});
-    helper.run(query);
-    assertTrue(methods.isEmpty());
-  }
-
-  @Test
-  public void testNewKeyFactory() throws Exception {
-    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
-    DatastoreRpc datastoreRpc = createStrictMock(DatastoreRpc.class);
-    DatastoreServiceOptions options =
-        DatastoreServiceOptions.builder().normalizeDataset(false).datastoreRpc(datastoreRpc)
-            .dataset("ds").build();
-    expect(datastoreService.options()).andReturn(options).atLeastOnce();
-    replay(datastoreRpc, datastoreService);
-    DatastoreHelper helper = DatastoreHelper.createFor(datastoreService);
-    KeyFactory keyFactory = helper.newKeyFactory();
-    assertSame(datastoreService, keyFactory.datastore());
-    verify(datastoreRpc, datastoreService);
-  }
-
   @Test
   public void testFetch() throws Exception {
     DatastoreService datastoreService = createStrictMock(DatastoreService.class);
@@ -130,8 +42,7 @@ public void testFetch() throws Exception {
     Entity entity2 = Entity.builder(key2).build();
     expect(datastoreService.get(key1, key2)).andReturn(Iterators.forArray(entity1, entity2)).once();
     replay(datastoreService);
-    DatastoreHelper helper = DatastoreHelper.createFor(datastoreService);
-    List values = helper.fetch(key1, key2);
+    List values = DatastoreHelper.fetch(datastoreService, key1, key2);
     assertEquals(2, values.size());
     assertEquals(entity1, values.get(0));
     assertEquals(entity2, values.get(1));
@@ -147,8 +58,7 @@ public void testRunInTransaction() throws Exception {
     expect(transaction.commit()).andReturn(null).once();
     expect(transaction.active()).andReturn(false).once();
     replay(datastoreService, transaction);
-    DatastoreHelper helper = DatastoreHelper.createFor(datastoreService);
-    helper.runInTransaction(new DatastoreHelper.RunInTransaction() {
+    DatastoreHelper.runInTransaction(datastoreService, new DatastoreService.RunInTransaction() {
       @Override
       public void run(DatastoreReaderWriter readerWriter) {
         assertTrue(transaction.active());
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index c5c5613a4bbc..c519a4330353 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -85,7 +85,6 @@ public class DatastoreServiceTest {
 
   private DatastoreServiceOptions options;
   private DatastoreService datastore;
-  private DatastoreHelper helper;
 
   private static LocalGcdHelper gcdHelper;
 
@@ -103,7 +102,6 @@ public void setUp() throws IOException, InterruptedException {
         .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
     datastore = DatastoreServiceFactory.getDefault(options);
-    helper = DatastoreHelper.createFor(datastore);
     // Prepare data for testing
     datastore.delete(KEY1, KEY2, KEY3, KEY4, KEY5);
     datastore.add(ENTITY1, ENTITY2);
@@ -133,7 +131,7 @@ public void testNewTransactionCommit() {
     transaction.delete(KEY1);
     transaction.commit();
 
-    List list = helper.fetch(KEY1, KEY2, KEY3);
+    List list = datastore.fetch(KEY1, KEY2, KEY3);
     assertNull(list.get(0));
     assertEquals(entity2, list.get(1));
     assertEquals(ENTITY3, list.get(2));
@@ -223,7 +221,7 @@ public void testNewTransactionRollback() {
 
     verifyNotUsable(transaction);
 
-    List list = helper.fetch(KEY1, KEY2, KEY3);
+    List list = datastore.fetch(KEY1, KEY2, KEY3);
     assertEquals(ENTITY1, list.get(0));
     assertEquals(ENTITY2, list.get(1));
     assertNull(list.get(2));
@@ -274,7 +272,7 @@ public void testNewBatch() {
 
     Batch.Response response = batch.submit();
     Iterator entities =
-        helper.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
+        datastore.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(entity1, entities.next());
     assertEquals(entity2, entities.next());
     assertEquals(ENTITY3, entities.next());
@@ -297,7 +295,7 @@ public void testNewBatch() {
     batch.delete(entity4.key(), entity5.key());
     batch.update(ENTITY1, ENTITY2, ENTITY3);
     batch.submit();
-    entities = helper.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
+    entities = datastore.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
     assertEquals(ENTITY1, entities.next());
     assertEquals(ENTITY2, entities.next());
     assertEquals(ENTITY3, entities.next());
@@ -429,7 +427,7 @@ public void testRunStructuredQuery() {
 
   @Test
   public void testAllocateId() {
-    KeyFactory keyFactory = helper.newKeyFactory().kind(KIND1);
+    KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     PartialKey pk1 = keyFactory.newKey();
     Key key1 = keyFactory.allocateId();
     assertEquals(key1.dataset(), pk1.dataset());
@@ -451,7 +449,7 @@ public void testAllocateId() {
 
   @Test
   public void testAllocateIdArray() {
-    KeyFactory keyFactory = helper.newKeyFactory().kind(KIND1);
+    KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     PartialKey partialKey1 = keyFactory.newKey();
     PartialKey partialKey2 = keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
     Key key3 = keyFactory.newKey("name");
@@ -492,7 +490,7 @@ public void testGet() {
   public void testGetArray() {
     datastore.put(ENTITY3);
     Iterator result =
-        helper.fetch(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3).iterator();
+        datastore.fetch(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3).iterator();
     assertEquals(ENTITY1, result.next());
     assertNull(result.next());
     assertEquals(ENTITY2, result.next());
@@ -520,7 +518,7 @@ public void testGetArray() {
 
   @Test
   public void testAddEntity() {
-    List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
+    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -537,7 +535,7 @@ public void testAddEntity() {
 
   @Test
   public void testAddPartialEntity() {
-    List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
+    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -568,7 +566,7 @@ public void testAddPartialEntity() {
 
   @Test
   public void testUpdate() {
-    List keys = helper.fetch(ENTITY1.key(), ENTITY3.key());
+    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -589,7 +587,7 @@ public void testUpdate() {
 
   @Test
   public void testPut() {
-    Iterator keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
@@ -598,7 +596,7 @@ public void testPut() {
     Entity entity2 = Entity.builder(ENTITY2).clear().set("bla", new NullValue()).build();
     assertNotEquals(ENTITY2, entity2);
     datastore.put(ENTITY3, ENTITY1, entity2);
-    keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(entity2, keys.next());
     assertEquals(ENTITY3, keys.next());
@@ -607,13 +605,13 @@ public void testPut() {
 
   @Test
   public void testDelete() {
-    Iterator keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
     assertFalse(keys.hasNext());
     datastore.delete(ENTITY1.key(), ENTITY2.key(), ENTITY3.key());
-    keys = helper.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertNull(keys.next());
     assertNull(keys.next());
     assertNull(keys.next());
@@ -622,7 +620,7 @@ public void testDelete() {
 
   @Test
   public void testKeyFactory() {
-    KeyFactory keyFactory = new KeyFactory(datastore).kind(KIND1);
+    KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     assertEquals(PARTIAL_KEY1, keyFactory.newKey());
     assertEquals(PartialKey.builder(PARTIAL_KEY1).kind(KIND2).build(),
         new KeyFactory(datastore).kind(KIND2).newKey());

From ce8f4a1b5e69f3f7d85716a1e895085136e93c3a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 3 Mar 2015 17:36:54 -0800
Subject: [PATCH 127/732] wip

---
 .../java/com/google/gcloud/storage/Acl.java   | 40 ++++++++++++++++---
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 280139035745..7cd251d5b43d 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -38,11 +38,33 @@ public enum Type {
     PROJECT
   }
 
+  public static class Builder {
+    private Builder() {
+
+    }
+
+    Acl build()
+  }
+
+  public static class Domain extends Acl {
+
+    private final String domain;
+
+    Domain(Role role, String domain) {
+      super(Type.USER, role);
+      this.email = email;
+    }
+
+    public static User domain(Role role, String domain) {
+      return new User(role, email);
+    }
+  }
+
   public static class User extends Acl {
 
     private static final long serialVersionUID = 3076518036392737008L;
 
-    private String email;
+    private final String email;
 
     User(Role role, String email) {
       super(Type.USER, role);
@@ -53,20 +75,20 @@ String email() {
       return email;
     }
 
-    public static User forEmail(Role role, String email) {
+    public static User email(Role role, String email) {
       return new User(role, email);
     }
 
     public static User allUsers(Role role) {
-      return forEmail(role, "allUsers");
+      return email(role, "allUsers");
     }
 
     public static User allAuthenticatedUsers(Role role) {
-      return forEmail(role, "allAuthenticatedUsers");
+      return email(role, "allAuthenticatedUsers");
     }
   }
 
-  
+
   Acl(Type type, Role role) {
     this.type = type;
     this.role = role;
@@ -80,4 +102,12 @@ public Type type() {
   public Role role() {
     return role;
   }
+
+  public Builder toBuilder() {
+
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
 }

From 0f252cf76c7de0a98e518c7d952db426d2281bfa Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 5 Mar 2015 18:17:18 -0800
Subject: [PATCH 128/732] Datastore team code review changes

---
 .../com/google/gcloud/datastore/Blob.java     | 14 ++---
 .../google/gcloud/datastore/BlobValue.java    |  2 +-
 .../gcloud/datastore/DatastoreHelper.java     | 10 ++--
 .../gcloud/datastore/DatastoreReader.java     |  2 +-
 .../gcloud/datastore/DatastoreService.java    | 25 ++++++---
 .../datastore/DatastoreServiceException.java  |  4 ++
 .../datastore/DatastoreServiceImpl.java       | 14 ++---
 .../google/gcloud/datastore/EntityValue.java  |  2 +-
 .../com/google/gcloud/datastore/GqlQuery.java |  8 +--
 .../google/gcloud/datastore/ListValue.java    |  3 +-
 .../gcloud/datastore/ProjectionEntity.java    |  6 +--
 .../{QueryResult.java => QueryResults.java}   |  6 +--
 ...yResultImpl.java => QueryResultsImpl.java} | 10 ++--
 .../google/gcloud/datastore/StringValue.java  |  4 --
 .../gcloud/datastore/StructuredQuery.java     |  4 +-
 .../google/gcloud/datastore/Transaction.java  |  2 +-
 .../gcloud/datastore/TransactionImpl.java     |  4 +-
 .../google/gcloud/datastore/Validator.java    |  5 +-
 .../com/google/gcloud/datastore/Value.java    | 12 ++---
 .../gcloud/datastore/DatastoreHelperTest.java | 52 ++++++++++++++-----
 .../datastore/DatastoreServiceTest.java       | 26 +++++-----
 .../google/gcloud/datastore/ValueTest.java    |  3 ++
 22 files changed, 127 insertions(+), 91 deletions(-)
 rename src/main/java/com/google/gcloud/datastore/{QueryResult.java => QueryResults.java} (88%)
 rename src/main/java/com/google/gcloud/datastore/{QueryResultImpl.java => QueryResultsImpl.java} (91%)

diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/src/main/java/com/google/gcloud/datastore/Blob.java
index 42b79cc3650d..5a759240be38 100644
--- a/src/main/java/com/google/gcloud/datastore/Blob.java
+++ b/src/main/java/com/google/gcloud/datastore/Blob.java
@@ -16,7 +16,6 @@
 
 package com.google.gcloud.datastore;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.services.datastore.DatastoreV1;
@@ -34,7 +33,6 @@
 
 /**
  * A Google Cloud Datastore Blob.
- * A Datastore blob is limited to {@value #MAX_LENGTH} bytes.
  * This class is immutable.
  *
  * @see Google Cloud Datastore Entities, Properties, and Keys
@@ -42,15 +40,11 @@
 public final class Blob extends Serializable {
 
   private static final long serialVersionUID = 3835421019618247721L;
-  public static final int MAX_LENGTH = 1_000_000;
 
   private final transient ByteString byteString;
 
-  Blob(ByteString byteString, boolean enforceLimits) {
+  Blob(ByteString byteString) {
     this.byteString = checkNotNull(byteString);
-    if (enforceLimits) {
-      checkArgument(byteString.size() <= MAX_LENGTH, "May be a maximum of %,d bytes", MAX_LENGTH);
-    }
   }
 
   @Override
@@ -134,11 +128,11 @@ ByteString byteString() {
   }
 
   public static Blob copyFrom(byte[] bytes) {
-    return new Blob(ByteString.copyFrom(bytes), true);
+    return new Blob(ByteString.copyFrom(bytes));
   }
 
   public static Blob copyFrom(ByteBuffer bytes) {
-    return new Blob(ByteString.copyFrom(bytes), true);
+    return new Blob(ByteString.copyFrom(bytes));
   }
 
   public static Blob copyFrom(InputStream input) throws IOException {
@@ -158,6 +152,6 @@ protected Value toPb() {
 
   @Override
   protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    return new Blob(DatastoreV1.Value.parseFrom(bytesPb).getBlobValue(), false);
+    return new Blob(DatastoreV1.Value.parseFrom(bytesPb).getBlobValue());
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java
index 4dcb80163ae4..1323986eb485 100644
--- a/src/main/java/com/google/gcloud/datastore/BlobValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java
@@ -41,7 +41,7 @@ public Builder newBuilder(Blob value) {
 
         @Override
         protected Blob getValue(DatastoreV1.Value from) {
-          return new Blob(from.getBlobValue(), false);
+          return new Blob(from.getBlobValue());
         }
 
         @Override
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index bca038e88b35..f650f4e44b3a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -51,12 +51,16 @@ static List fetch(DatastoreReader reader, Key... keys) {
     return list;
   }
 
-  static void runInTransaction(DatastoreService datastoreService,
-      DatastoreService.RunInTransaction runFor, TransactionOption... options) {
+  static  T runInTransaction(DatastoreService datastoreService,
+      DatastoreService.TransactionCallable callable, TransactionOption... options) {
     Transaction transaction = datastoreService.newTransaction(options);
     try {
-      runFor.run(transaction);
+      T value = callable.run(transaction);
       transaction.commit();
+      return value;
+    } catch (Exception ex) {
+      transaction.rollback();
+      throw DatastoreServiceException.propagateUserException(ex);
     } finally {
       if (transaction.active()) {
         transaction.rollback();
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index 7b24dbba3f25..b1dc0a70d392 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -55,5 +55,5 @@ public interface DatastoreReader {
    *
    * @throws DatastoreServiceException upon failure.
    */
-   QueryResult run(Query query);
+   QueryResults run(Query query);
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index 395ceb2fc873..dd47c3a964c0 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -32,21 +32,32 @@ public interface DatastoreService extends Service, Data
    */
   Transaction newTransaction(TransactionOption... options);
 
-  interface RunInTransaction {
-    void run(DatastoreReaderWriter readerWriter);
+
+  /**
+   * An Callback for running with a Transactional
+   * {@link com.google.gcloud.datastore.DatastoreReaderWriter}.
+   * The associated transaction will be committed after a successful return from the {@code run}
+   * method. Any propagated exception will cause the transaction to be rolled-back.
+   *
+   * @param  the type of the return value
+   */
+  interface TransactionCallable {
+    T run(DatastoreReaderWriter readerWriter) throws Exception;
   }
 
 
   /**
-   * Invokes the callback's {@link RunInTransaction#run} method with a
+   * Invokes the callback's {@link DatastoreService.TransactionCallable#run} method with a
    * {@link DatastoreReaderWriter} that is associated with a new transaction.
-   * The transaction will be committed upon successful invocation or rollback
-   * otherwise.
+   * The transaction will be committed upon successful invocation.
+   * Any thrown exception will cause the transaction to rollback and will be propagated
+   * as a {@link DatastoreServiceException} with the original exception as its root cause.
    *
-   * @param runFor the functor to call with the transactional readerWriter
+   * @param callable the callback to call with a newly created transactional readerWriter
    * @param options the options for the created transaction
+   * @throws DatastoreServiceException upon failure
    */
-  void runInTransaction(RunInTransaction runFor, TransactionOption... options);
+   T runInTransaction(TransactionCallable callable, TransactionOption... options);
 
   /**
    * Returns a new Batch for processing multiple write operations in one request.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index c365745863de..14301a21dc86 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -148,4 +148,8 @@ static DatastoreServiceException translateAndThrow(DatastoreRpcException excepti
   static DatastoreServiceException throwInvalidRequest(String massage, Object... params) {
     throw new DatastoreServiceException(Code.FAILED_PRECONDITION, String.format(massage, params));
   }
+
+  static DatastoreServiceException propagateUserException(Exception ex) {
+    throw new DatastoreServiceException(Code.UNKNOWN, ex.getMessage(), ex);
+  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 52430a61c86d..f61e0c56fe5a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -90,17 +90,18 @@ public Transaction newTransaction(TransactionOption... options) {
     return new TransactionImpl(this, options);
   }
 
-  public void runInTransaction(RunInTransaction runFor, TransactionOption... options) {
-    DatastoreHelper.runInTransaction(this, runFor, options);
+  @Override
+  public  T runInTransaction(TransactionCallable callable, TransactionOption... options) {
+    return DatastoreHelper.runInTransaction(this, callable, options);
   }
 
   @Override
-  public  QueryResult run(Query query) {
+  public  QueryResults run(Query query) {
     return run(null, query);
   }
 
-   QueryResult run(DatastoreV1.ReadOptions readOptionsPb, Query query) {
-    return new QueryResultImpl<>(this, readOptionsPb, query);
+   QueryResults run(DatastoreV1.ReadOptions readOptionsPb, Query query) {
+    return new QueryResultsImpl<>(this, readOptionsPb, query);
   }
 
   DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestPb) {
@@ -129,10 +130,9 @@ public List allocateId(PartialKey... keys) {
     for (PartialKey key : keys) {
       requestPb.addKey(trimNameOrId(key).toPb());
     }
-    // TODO(ozarov): will need to populate "force" after b/18594027 is fixed.
     DatastoreV1.AllocateIdsResponse responsePb = allocateIds(requestPb.build());
     Iterator keyIterator = responsePb.getKeyList().iterator();
-    ImmutableList.Builder builder = ImmutableList.builder().addAll(
+    ImmutableList.Builder builder = ImmutableList.builder().addAll(
         Iterators.transform(keyIterator, new Function() {
           @Override
           public Key apply(DatastoreV1.Key keyPb) {
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java
index cdf24d3ca8ed..071375e78d16 100644
--- a/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -59,7 +59,7 @@ private Builder() {
 
     @Override
     public Builder indexed(boolean indexed) {
-      // see b/8730533
+      // see issue #25
       Preconditions.checkArgument(!indexed, "EntityValue can't be indexed");
       return super.indexed(indexed);
     }
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 7ab0831c7e30..3260a309ea6a 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -45,7 +45,7 @@
  * 
When the type of the results is known the preferred usage would be:
  * 
{@code
  *   Query<Entity> query = GqlQuery.builder(Query.Type.FULL, "select * from kind").build();
- *   QueryResult<Entity> results = datastore.run(query);
+ *   QueryResults<Entity> results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -55,9 +55,9 @@
  * 
When the type of the results is unknown you can use this approach:
  * 
{@code
  *   Query<?> query = GqlQuery.builder("select __key__ from kind").build();
- *   QueryResult<?> results = datastore.run(query);
+ *   QueryResults<?> results = datastore.run(query);
  *   if (Key.class.isAssignableFrom(results.resultClass())) {
- *     QueryResult<Key> keys = (QueryResult<Key>) results;
+ *     QueryResults<Key> keys = (QueryResults<Key>) results;
  *     while (keys.hasNext()) {
  *       Key key = keys.next();
  *       ...
@@ -389,7 +389,7 @@ protected void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb) {
 
   @Override
   protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
-    // See b/18705483
+    // See issue #17
     throw new UnsupportedOperationException("paging for this query is not implemented yet");
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java
index abf7ff9c9a40..9314297e8a3a 100644
--- a/src/main/java/com/google/gcloud/datastore/ListValue.java
+++ b/src/main/java/com/google/gcloud/datastore/ListValue.java
@@ -71,6 +71,7 @@ private Builder() {
     }
 
     public Builder addValue(Value value) {
+      // see datastore_v1.proto definition for list_value
       Preconditions.checkArgument(value.type() != Type.LIST, "Cannot contain another list");
       listBuilder.add(value);
       return this;
@@ -86,7 +87,7 @@ public Builder addValue(Value first, Value... other) {
 
     @Override
     public Builder indexed(boolean indexed) {
-      // see b/18704917
+      // see issue #26
       throw DatastoreServiceException.throwInvalidRequest("ListValue can't specify index");
     }
 
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
index decbbd547298..17d260332eb6 100644
--- a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -94,7 +94,7 @@ public Key key() {
     return key;
   }
 
-  @SuppressWarnings("unchecked")
+  @SuppressWarnings({"unchecked", "deprecation"})
   @Override
   public DateTime getDateTime(String name) {
     Value value = getValue(name);
@@ -104,12 +104,12 @@ public DateTime getDateTime(String name) {
     return ((Value) value).get();
   }
 
-  @SuppressWarnings("unchecked")
+  @SuppressWarnings({"unchecked", "deprecation"})
   @Override
   public Blob getBlob(String name) {
     Value value = getValue(name);
     if (value.hasMeaning() && value.meaning() == 18 && value instanceof StringValue) {
-      return new Blob(ByteString.copyFromUtf8(getString(name)), false);
+      return new Blob(ByteString.copyFromUtf8(getString(name)));
     }
     return ((Value) value).get();
   }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResult.java b/src/main/java/com/google/gcloud/datastore/QueryResults.java
similarity index 88%
rename from src/main/java/com/google/gcloud/datastore/QueryResult.java
rename to src/main/java/com/google/gcloud/datastore/QueryResults.java
index 45cca8c13dd3..6310eec93215 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResult.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResults.java
@@ -27,7 +27,7 @@
  *
  * @param  the type of the results value.
  */
-public interface QueryResult extends Iterator {
+public interface QueryResults extends Iterator {
 
   /**
    * Returns the actual class of the result's values.
@@ -35,8 +35,8 @@ public interface QueryResult extends Iterator {
   Class resultClass();
 
   /**
-   * Returns the Cursor for the {@link #next} result.
+   * Returns the Cursor for point after the value returned in the last {@link #next} call.
    * Not currently implemented (depends on v1beta3).
    */
-  Cursor cursor();
+  Cursor cursorAfter();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
similarity index 91%
rename from src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
rename to src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
index 2944724e116b..324a63d7a2a5 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
@@ -25,7 +25,7 @@
 import java.util.Iterator;
 import java.util.Objects;
 
-class QueryResultImpl extends AbstractIterator implements QueryResult {
+class QueryResultsImpl extends AbstractIterator implements QueryResults {
 
   private final DatastoreServiceImpl datastore;
   private final DatastoreV1.ReadOptions readOptionsPb;
@@ -39,8 +39,8 @@ class QueryResultImpl extends AbstractIterator implements QueryResult {
   //private ByteString cursor; // only available in v1beta3
 
 
-  QueryResultImpl(DatastoreServiceImpl datastore, DatastoreV1.ReadOptions readOptionsPb,
-      Query query) {
+  QueryResultsImpl(DatastoreServiceImpl datastore, DatastoreV1.ReadOptions readOptionsPb,
+                   Query query) {
     this.datastore = datastore;
     this.readOptionsPb = readOptionsPb;
     this.query = query;
@@ -66,7 +66,7 @@ private void sendRequest() {
     queryResultBatchPb = datastore.runQuery(requestPb.build()).getBatch();
     lastBatch = queryResultBatchPb.getMoreResults() != MoreResultsType.NOT_FINISHED;
     entityResultPbIter = queryResultBatchPb.getEntityResultList().iterator();
-    // cursor = resultPb.getSkippedCursor(); // only available in v1beta3
+    // cursor = resultPb.getSkippedCursor(); // available in v1beta3, use startCursor if not skipped
     actualType = Type.fromPb(queryResultBatchPb.getEntityResultType());
     if (Objects.equals(queryType, Type.PROJECTION)) {
       // projection entity can represent all type of results
@@ -97,7 +97,7 @@ public Class resultClass() {
   }
 
   @Override
-  public Cursor cursor() {
+  public Cursor cursorAfter() {
     //return new Cursor(cursor); // only available in v1beta3
     return null;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java
index 699d2bd97cd7..5a944da08205 100644
--- a/src/main/java/com/google/gcloud/datastore/StringValue.java
+++ b/src/main/java/com/google/gcloud/datastore/StringValue.java
@@ -17,7 +17,6 @@
 package com.google.gcloud.datastore;
 
 import static com.google.api.services.datastore.DatastoreV1.Value.STRING_VALUE_FIELD_NUMBER;
-import static com.google.common.base.Preconditions.checkArgument;
 
 import com.google.api.services.datastore.DatastoreV1;
 
@@ -59,9 +58,6 @@ private Builder() {
 
     @Override
     public StringValue build() {
-      if (Boolean.TRUE.equals(getIndexed())) {
-        checkArgument(get().length() <= 500, "Indexed string is limited to 500 characters");
-      }
       return new StringValue(this);
     }
   }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 03372fe3d96f..61c83d734da4 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -48,7 +48,7 @@
  * 
A simple query that returns all entities for a specific kind
  * 
 {@code
  *   StructuredQuery query = StructuredQuery.builder().kind(kind).build();
- *   QueryResult results = datastore.run(query);
+ *   QueryResults results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -66,7 +66,7 @@
  *       .orderBy(OrderBy.asc("age"))
  *       .limit(10)
  *       .build();
- *   QueryResult results = datastore.run(query);
+ *   QueryResults results = datastore.run(query);
  *   ...
  * } 

  *
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index c749a007f800..d2ff472aa2a9 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -98,7 +98,7 @@ interface Response {
    * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
-   QueryResult run(Query query);
+   QueryResults run(Query query);
 
   /**
    * Commit the transaction.
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 2118153c252d..99a415d0d3aa 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -66,7 +66,7 @@ public List generatedKeys() {
       requestPb.setIsolationLevel(isolationLevel.level().toPb());
     }
     ForceWrites forceWrites = (ForceWrites) optionsMap.get(TransactionOption.ForceWrites.class);
-    force = forceWrites == null ? false : forceWrites.force();
+    force = forceWrites != null && forceWrites.force();
     transaction = datastore.requestTransactionId(requestPb);
   }
 
@@ -90,7 +90,7 @@ public List fetch(Key... keys) {
   }
 
   @Override
-  public  QueryResult run(Query query) {
+  public  QueryResults run(Query query) {
     validateActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/src/main/java/com/google/gcloud/datastore/Validator.java
index 9ee6a5e57d68..055a8ff4ebdc 100644
--- a/src/main/java/com/google/gcloud/datastore/Validator.java
+++ b/src/main/java/com/google/gcloud/datastore/Validator.java
@@ -47,7 +47,7 @@ static String validateDataset(String dataset) {
   static String validateNamespace(String namespace) {
     if (namespace != null) {
       checkArgument(!namespace.isEmpty(), "namespace must not be an empty string");
-      checkArgument(namespace.length() <= 100,
+      checkArgument(namespace.length() <= MAX_NAMESPACE_LENGTH,
           "namespace must not contain more than 100 characters");
       checkArgument(NAMESPACE_PATTERN.matcher(namespace).matches(),
           "namespace must the following pattern: " + NAMESPACE_PATTERN.pattern());
@@ -57,7 +57,8 @@ static String validateNamespace(String namespace) {
 
   static String validateKind(String kind) {
     checkArgument(!Strings.isNullOrEmpty(kind), "kind must not be empty or null");
-    checkArgument(kind.length() <= 500, "kind must not contain more than 500 characters");
     return kind;
   }
+
+
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index 2db172039535..300321bbefa4 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -16,7 +16,6 @@
 
 package com.google.gcloud.datastore;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.services.datastore.DatastoreV1;
@@ -136,6 +135,7 @@ interface Builder, B extends Builder> {
 
     Integer getMeaning();
 
+    @Deprecated
     B meaning(Integer meaning);
 
     V get();
@@ -266,14 +266,6 @@ 
, B extends BaseBuilder> Value(Builder buil
     type = builder.getType();
     indexed = builder.getIndexed();
     meaning = builder.getMeaning();
-    // some validations:
-    if (meaning != null && indexed != null) {
-      // TODO: consider supplying Ranges for allowed meaning and validating it here
-      // more specific validation could be done on the specific types themselves
-      // upon construction [e.g. integer with a meaning 13 should be in the range [0,100]]
-      checkArgument(!indexed || meaning != 15 && meaning != 22,
-          "Indexed values should not have meaning with 15 or 22");
-    }
     value = builder.get();
   }
 
@@ -289,10 +281,12 @@ public final Boolean indexed() {
     return indexed;
   }
 
+  @Deprecated
   public final boolean hasMeaning() {
     return meaning != null;
   }
 
+  @Deprecated
   public final Integer meaning() {
     return meaning;
   }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 580114780a90..71433d7f178f 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -17,15 +17,14 @@
 package com.google.gcloud.datastore;
 
 import static junit.framework.TestCase.assertTrue;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.verify;
+import static junit.framework.TestCase.fail;
+import static org.easymock.EasyMock.*;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertSame;
 
 import com.google.common.collect.Iterators;
-
+import com.google.gcloud.datastore.DatastoreService.TransactionCallable;
+import org.easymock.EasyMock;
 import org.junit.Test;
 
 import java.util.List;
@@ -58,13 +57,42 @@ public void testRunInTransaction() throws Exception {
     expect(transaction.commit()).andReturn(null).once();
     expect(transaction.active()).andReturn(false).once();
     replay(datastoreService, transaction);
-    DatastoreHelper.runInTransaction(datastoreService, new DatastoreService.RunInTransaction() {
-      @Override
-      public void run(DatastoreReaderWriter readerWriter) {
-        assertTrue(transaction.active());
-        assertSame(transaction, readerWriter);
-      }
-    });
+    String value = DatastoreHelper.runInTransaction(datastoreService,
+        new TransactionCallable() {
+          @Override
+          public String run(DatastoreReaderWriter readerWriter) {
+            assertTrue(transaction.active());
+            assertSame(transaction, readerWriter);
+            return "done";
+          }
+        });
+    verify(datastoreService, transaction);
+    assertEquals("done", value);
+  }
+
+  @Test
+  public void testRunInTransactionWithException() throws Exception {
+    final DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    final Transaction transaction = createStrictMock(Transaction.class);
+    expect(datastoreService.newTransaction()).andReturn(transaction).once();
+    expect(transaction.active()).andReturn(true).once();
+    transaction.rollback();
+    EasyMock.expectLastCall().once();
+    expect(transaction.active()).andReturn(false).once();
+    replay(datastoreService, transaction);
+    try {
+      DatastoreHelper.runInTransaction(datastoreService, new TransactionCallable() {
+        @Override
+        public Void run(DatastoreReaderWriter readerWriter) throws Exception {
+          assertTrue(transaction.active());
+          assertSame(transaction, readerWriter);
+          throw new Exception("Bla");
+        }
+      });
+      fail("DatastoreServiceException was expected");
+    } catch (DatastoreServiceException ex) {
+      assertEquals("Bla", ex.getCause().getMessage());
+    }
     verify(datastoreService, transaction);
   }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index c519a4330353..e16572d683ad 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -180,7 +180,7 @@ public void testTransactionWithQuery() {
     Query query =
         StructuredQuery.builder().kind(KIND2).filter(PropertyFilter.hasAncestor(KEY2)).build();
     Transaction transaction = datastore.newTransaction();
-    QueryResult results = transaction.run(query);
+    QueryResults results = transaction.run(query);
     assertEquals(ENTITY2, results.next());
     assertFalse(results.hasNext());
     transaction.add(ENTITY3);
@@ -313,7 +313,7 @@ public void testNewBatch() {
   @Test
   public void testRunGqlQueryNoCasting() {
     Query query1 = GqlQuery.builder(Type.FULL, "select * from " + KIND1).build();
-    QueryResult results1 = datastore.run(query1);
+    QueryResults results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
@@ -321,7 +321,7 @@ public void testRunGqlQueryNoCasting() {
     datastore.put(ENTITY3);
     Query query2 =  GqlQuery.builder(
         Type.FULL, "select * from " + KIND2 + " order by __key__").build();
-    QueryResult results2 = datastore.run(query2);
+    QueryResults results2 = datastore.run(query2);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY2, results2.next());
     assertTrue(results2.hasNext());
@@ -334,14 +334,14 @@ public void testRunGqlQueryNoCasting() {
 
     Query keyOnlyQuery =
         GqlQuery.builder(Type.KEY_ONLY, "select __key__ from " + KIND1).build();
-    QueryResult keyOnlyResults = datastore.run(keyOnlyQuery);
+    QueryResults keyOnlyResults = datastore.run(keyOnlyQuery);
     assertTrue(keyOnlyResults.hasNext());
     assertEquals(KEY1, keyOnlyResults.next());
     assertFalse(keyOnlyResults.hasNext());
 
     GqlQuery keyProjectionQuery = GqlQuery.builder(
         Type.PROJECTION, "select __key__ from " + KIND1).build();
-    QueryResult keyProjectionResult = datastore.run(keyProjectionQuery);
+    QueryResults keyProjectionResult = datastore.run(keyProjectionQuery);
     assertTrue(keyProjectionResult.hasNext());
     ProjectionEntity projectionEntity = keyProjectionResult.next();
     assertEquals(KEY1, projectionEntity.key());
@@ -351,7 +351,7 @@ public void testRunGqlQueryNoCasting() {
     GqlQuery projectionQuery = GqlQuery.builder(
         Type.PROJECTION, "select str, date from " + KIND1).build();
 
-    QueryResult projectionResult = datastore.run(projectionQuery);
+    QueryResults projectionResult = datastore.run(projectionQuery);
     assertTrue(projectionResult.hasNext());
     projectionEntity = projectionResult.next();
     assertEquals("str", projectionEntity.getString("str"));
@@ -366,16 +366,16 @@ public void testRunGqlQueryNoCasting() {
   public void testRunGqlQueryWithCasting() {
     @SuppressWarnings("unchecked")
     Query query1 = (Query) GqlQuery.builder("select * from " + KIND1).build();
-    QueryResult results1 = datastore.run(query1);
+    QueryResults results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
     Query query2 = GqlQuery.builder("select * from " + KIND1).build();
-    QueryResult results2 = datastore.run(query2);
+    QueryResults results2 = datastore.run(query2);
     assertSame(Entity.class, results2.resultClass());
     @SuppressWarnings("unchecked")
-    QueryResult results3 = (QueryResult) results2;
+    QueryResults results3 = (QueryResults) results2;
     assertTrue(results3.hasNext());
     assertEquals(ENTITY1, results3.next());
     assertFalse(results3.hasNext());
@@ -385,20 +385,20 @@ public void testRunGqlQueryWithCasting() {
   public void testRunStructuredQuery() {
     StructuredQuery query =
         StructuredQuery.builder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
-    QueryResult results1 = datastore.run(query);
+    QueryResults results1 = datastore.run(query);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
     StructuredQuery keyOnlyQuery =  StructuredQuery.keyOnlyBuilder().kind(KIND1).build();
-    QueryResult results2 = datastore.run(keyOnlyQuery);
+    QueryResults results2 = datastore.run(keyOnlyQuery);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY1.key(), results2.next());
     assertFalse(results2.hasNext());
 
     StructuredQuery keyOnlyProjectionQuery = StructuredQuery.projectionBuilder()
         .kind(KIND1).projection(Projection.property("__key__")).build();
-    QueryResult results3 = datastore.run(keyOnlyProjectionQuery);
+    QueryResults results3 = datastore.run(keyOnlyProjectionQuery);
     assertTrue(results3.hasNext());
     ProjectionEntity projectionEntity = results3.next();
     assertEquals(ENTITY1.key(), projectionEntity.key());
@@ -414,7 +414,7 @@ public void testRunStructuredQuery() {
         .limit(10)
         .build();
 
-    QueryResult results4 = datastore.run(projectionQuery);
+    QueryResults results4 = datastore.run(projectionQuery);
     assertTrue(results4.hasNext());
     ProjectionEntity entity = results4.next();
     assertEquals(ENTITY2.key(), entity.key());
diff --git a/src/test/java/com/google/gcloud/datastore/ValueTest.java b/src/test/java/com/google/gcloud/datastore/ValueTest.java
index 161910126156..c9805a349dd9 100644
--- a/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -157,6 +157,7 @@ public void testIndexed() throws Exception {
     assertTrue(builder.indexed(true).build().indexed());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testHasMeaning() throws Exception {
     for (Value value: typeToValue.values()) {
@@ -167,6 +168,7 @@ public void testHasMeaning() throws Exception {
     assertTrue(builder.meaning(10).build().hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testMeaning() throws Exception {
     for (Value value: typeToValue.values()) {
@@ -190,6 +192,7 @@ public void testGet() throws Exception {
     assertEquals(value, builder.set(value).build().get());
   }
 
+  @SuppressWarnings({"unchecked", "deprecation"})
   @Test
   public void testToBuilder() throws Exception {
     Set content = Collections.singleton("bla");

From 73978c08748293bfb81fb3de0c50f27f740f3544 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 6 Mar 2015 10:44:24 -0800
Subject: [PATCH 129/732] rename PartialKey to IncompleteKey

---
 .../gcloud/datastore/DatastoreService.java    |  6 ++---
 .../datastore/DatastoreServiceImpl.java       | 14 ++++++------
 .../{PartialKey.java => IncompleteKey.java}   | 16 +++++++-------
 .../java/com/google/gcloud/datastore/Key.java | 12 +++++-----
 .../google/gcloud/datastore/KeyFactory.java   |  6 ++---
 .../gcloud/datastore/PartialEntity.java       | 22 +++++++++----------
 .../BaseDatastoreBatchWriterTest.java         |  2 +-
 .../gcloud/datastore/BaseEntityTest.java      |  2 +-
 .../gcloud/datastore/DatastoreHelperTest.java |  2 +-
 .../datastore/DatastoreServiceTest.java       | 22 +++++++++----------
 .../google/gcloud/datastore/EntityTest.java   |  2 +-
 ...ialKeyTest.java => IncompleteKeyTest.java} | 10 ++++-----
 .../gcloud/datastore/KeyFactoryTest.java      | 16 +++++++-------
 .../gcloud/datastore/PartialEntityTest.java   |  2 +-
 .../gcloud/datastore/SerializationTest.java   |  8 +++----
 15 files changed, 71 insertions(+), 71 deletions(-)
 rename src/main/java/com/google/gcloud/datastore/{PartialKey.java => IncompleteKey.java} (86%)
 rename src/test/java/com/google/gcloud/datastore/{PartialKeyTest.java => IncompleteKeyTest.java} (80%)

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index dd47c3a964c0..7c678f6b3199 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -71,15 +71,15 @@ interface TransactionCallable {
    *
    * @throws DatastoreServiceException upon failure
    */
-  Key allocateId(PartialKey key);
+  Key allocateId(IncompleteKey key);
 
   /**
    * Returns a list of keys using the allocated ids ordered by the input.
    *
    * @throws DatastoreServiceException upon failure
-   * @see #allocateId(PartialKey)
+   * @see #allocateId(IncompleteKey)
    */
-  List allocateId(PartialKey... key);
+  List allocateId(IncompleteKey... key);
 
   /**
    * Datastore add operation.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index f61e0c56fe5a..00b2689c9ecc 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -117,17 +117,17 @@ DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestP
   }
 
   @Override
-  public Key allocateId(PartialKey key) {
-    return allocateId(new PartialKey[]{key}).get(0);
+  public Key allocateId(IncompleteKey key) {
+    return allocateId(new IncompleteKey[]{key}).get(0);
   }
 
   @Override
-  public List allocateId(PartialKey... keys) {
+  public List allocateId(IncompleteKey... keys) {
     if (keys.length == 0) {
       return Collections.emptyList();
     }
     DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder();
-    for (PartialKey key : keys) {
+    for (IncompleteKey key : keys) {
       requestPb.addKey(trimNameOrId(key).toPb());
     }
     DatastoreV1.AllocateIdsResponse responsePb = allocateIds(requestPb.build());
@@ -154,9 +154,9 @@ DatastoreV1.AllocateIdsResponse allocateIds(final DatastoreV1.AllocateIdsRequest
     }
   }
 
-  private PartialKey trimNameOrId(PartialKey key) {
+  private IncompleteKey trimNameOrId(IncompleteKey key) {
     if (key instanceof Key) {
-      return PartialKey.builder(key).build();
+      return IncompleteKey.builder(key).build();
     }
     return key;
   }
@@ -196,7 +196,7 @@ public List add(PartialEntity... entities) {
         commitResponse.getMutationResult().getInsertAutoIdKeyList().iterator();
     ImmutableList.Builder responseBuilder = ImmutableList.builder();
     for (PartialEntity entity : entities) {
-      PartialKey key = entity.key();
+      IncompleteKey key = entity.key();
       Entity completeEntity = completeEntities.get(key);
       if (completeEntity != null) {
         responseBuilder.add(completeEntity);
diff --git a/src/main/java/com/google/gcloud/datastore/PartialKey.java b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
similarity index 86%
rename from src/main/java/com/google/gcloud/datastore/PartialKey.java
rename to src/main/java/com/google/gcloud/datastore/IncompleteKey.java
index ae4c706f76c7..1da007f2092b 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialKey.java
+++ b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
@@ -28,7 +28,7 @@
  * Could be used as metadata for {@link PartialEntity}.
  * This class is immutable.
  */
-public class PartialKey extends BaseKey {
+public class IncompleteKey extends BaseKey {
 
   private static final long serialVersionUID = -75301206578793347L;
 
@@ -38,19 +38,19 @@ private Builder(String dataset, String kind) {
       super(dataset, kind);
     }
 
-    private Builder(PartialKey copyFrom) {
+    private Builder(IncompleteKey copyFrom) {
       super(copyFrom);
     }
 
     @Override
-    public PartialKey build() {
+    public IncompleteKey build() {
       ImmutableList path = ImmutableList.builder()
           .addAll(ancestors).add(PathElement.of(kind)).build();
-      return new PartialKey(dataset, namespace, path);
+      return new IncompleteKey(dataset, namespace, path);
     }
   }
 
-  PartialKey(String dataset, String namespace, ImmutableList path) {
+  IncompleteKey(String dataset, String namespace, ImmutableList path) {
     super(dataset, namespace, path);
   }
 
@@ -59,7 +59,7 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
     return fromPb(DatastoreV1.Key.parseFrom(bytesPb));
   }
 
-  static PartialKey fromPb(DatastoreV1.Key keyPb) {
+  static IncompleteKey fromPb(DatastoreV1.Key keyPb) {
     String dataset = null;
     String namespace = null;
     if (keyPb.hasPartitionId()) {
@@ -82,14 +82,14 @@ static PartialKey fromPb(DatastoreV1.Key keyPb) {
     if (leaf.nameOrId() != null) {
       return new Key(dataset, namespace, path);
     }
-    return new PartialKey(dataset, namespace, path);
+    return new IncompleteKey(dataset, namespace, path);
   }
 
   public static Builder builder(String dataset, String kind) {
     return new Builder(dataset, kind);
   }
 
-  public static Builder builder(PartialKey copyFrom) {
+  public static Builder builder(IncompleteKey copyFrom) {
     return new Builder(copyFrom);
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index 1555b109004c..e5f7ce6083b0 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -35,7 +35,7 @@
  *
  * @see Google Cloud Datastore Entities, Properties, and Keys
  */
-public final class Key extends PartialKey {
+public final class Key extends IncompleteKey {
 
   private static final long serialVersionUID = 3160994559785491356L;
 
@@ -54,12 +54,12 @@ private Builder(String dataset, String kind, long id) {
       this.id = id;
     }
 
-    private Builder(PartialKey copyFrom, String name) {
+    private Builder(IncompleteKey copyFrom, String name) {
       super(copyFrom);
       this.name = name;
     }
 
-    private Builder(PartialKey copyFrom, long id) {
+    private Builder(IncompleteKey copyFrom, long id) {
       super(copyFrom);
       this.id = id;
     }
@@ -168,7 +168,7 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   }
 
   static Key fromPb(DatastoreV1.Key keyPb) {
-    PartialKey key = PartialKey.fromPb(keyPb);
+    IncompleteKey key = IncompleteKey.fromPb(keyPb);
     Preconditions.checkState(key instanceof Key, "Key is not complete");
     return (Key) key;
   }
@@ -185,11 +185,11 @@ public static Builder builder(Key copyFrom) {
     return new Builder(copyFrom);
   }
 
-  public static Builder builder(PartialKey copyFrom, String name) {
+  public static Builder builder(IncompleteKey copyFrom, String name) {
     return new Builder(copyFrom, name);
   }
 
-  public static Builder builder(PartialKey copyFrom, long id) {
+  public static Builder builder(IncompleteKey copyFrom, long id) {
     return new Builder(copyFrom, id);
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index 645dd8eb3879..1bf14048f598 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -34,10 +34,10 @@ public KeyFactory(DatastoreService service) {
     namespace(service.options().namespace());
   }
 
-  public PartialKey newKey() {
+  public IncompleteKey newKey() {
     ImmutableList path = ImmutableList.builder()
         .addAll(ancestors).add(PathElement.of(kind)).build();
-    return new PartialKey(dataset, namespace, path);
+    return new IncompleteKey(dataset, namespace, path);
   }
 
   public Key newKey(String name) {
@@ -61,7 +61,7 @@ public Key allocateId() {
   }
 
   @Override
-  protected PartialKey build() {
+  protected IncompleteKey build() {
     return newKey();
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
index 1831d810e831..b8cb27f64abe 100644
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
@@ -33,11 +33,11 @@ public class PartialEntity extends BaseEntity {
 
   private static final long serialVersionUID = 6492561268709192891L;
 
-  private final transient PartialKey key;
+  private final transient IncompleteKey key;
 
   public static class Builder extends BaseEntity.Builder {
 
-    private PartialKey key;
+    private IncompleteKey key;
 
     private Builder() {
     }
@@ -47,7 +47,7 @@ private Builder(PartialEntity entity) {
       key = entity.key();
     }
 
-    public Builder key(PartialKey key) {
+    public Builder key(IncompleteKey key) {
       this.key = key;
       return this;
     }
@@ -58,7 +58,7 @@ public PartialEntity build() {
     }
   }
 
-  protected PartialEntity(PartialKey key, ImmutableSortedMap> properties) {
+  protected PartialEntity(IncompleteKey key, ImmutableSortedMap> properties) {
     super(properties);
     this.key = key;
   }
@@ -70,7 +70,7 @@ public boolean hasKey() {
   /**
    * Returns the key for this entity or {@code null} if it does not have one.
    */
-  public PartialKey key() {
+  public IncompleteKey key() {
     return key;
   }
 
@@ -110,21 +110,21 @@ static PartialEntity fromPb(DatastoreV1.Entity entityPb) {
     for (DatastoreV1.Property property : entityPb.getPropertyList()) {
       properties.put(property.getName(), Value.fromPb(property.getValue()));
     }
-    PartialKey partialKey = null;
+    IncompleteKey incompleteKey = null;
     if (entityPb.hasKey()) {
-      partialKey = PartialKey.fromPb(entityPb.getKey());
-      if (partialKey instanceof Key) {
-        return new Entity((Key) partialKey, properties.build());
+      incompleteKey = IncompleteKey.fromPb(entityPb.getKey());
+      if (incompleteKey instanceof Key) {
+        return new Entity((Key) incompleteKey, properties.build());
       }
     }
-    return new PartialEntity(partialKey, properties.build());
+    return new PartialEntity(incompleteKey, properties.build());
   }
 
   public static Builder builder() {
     return new Builder();
   }
 
-  public static Builder builder(PartialKey key) {
+  public static Builder builder(IncompleteKey key) {
     return new Builder().key(key);
   }
 
diff --git a/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
index 8d440407a9e4..e1987beec9f5 100644
--- a/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;

import org.junit.Before;
import org.junit.Test;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder("dataset1", "kind1", 1).build();
  private static final Key KEY3 = Key.builder("dataset1", "kind1", 2).build();
  private static final PartialKey PARTIAL_KEY = PartialKey.builder("dataset1", "kind1").build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final PartialEntity PARTIAL_ENTITY_1 = Entity.builder(PARTIAL_KEY).build();
  private static final PartialEntity PARTIAL_ENTITY_2 =
      PartialEntity.builder(PARTIAL_KEY).set("name", "dan").build();


  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    protected DatastoreBatchWriter() {
      super("test");
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @Test
  public void testAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(ENTITY2.toPb())
        .addInsert(ENTITY3.toPb())
        .build();
    batchWriter.add(ENTITY1, ENTITY2);
    batchWriter.add(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddAutoId() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(PARTIAL_ENTITY_1.toPb())
        .addInsertAutoId(PARTIAL_ENTITY_2.toPb())
        .build();
    batchWriter.add(ENTITY1, PARTIAL_ENTITY_1);
    batchWriter.add(PARTIAL_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAutoIdWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(PARTIAL_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(PARTIAL_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.add(PARTIAL_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;

import org.junit.Before;
import org.junit.Test;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder("dataset1", "kind1", 1).build();
  private static final Key KEY3 = Key.builder("dataset1", "kind1", 2).build();
  private static final IncompleteKey PARTIAL_KEY = IncompleteKey.builder("dataset1", "kind1").build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final PartialEntity PARTIAL_ENTITY_1 = Entity.builder(PARTIAL_KEY).build();
  private static final PartialEntity PARTIAL_ENTITY_2 =
      PartialEntity.builder(PARTIAL_KEY).set("name", "dan").build();


  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    protected DatastoreBatchWriter() {
      super("test");
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @Test
  public void testAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(ENTITY2.toPb())
        .addInsert(ENTITY3.toPb())
        .build();
    batchWriter.add(ENTITY1, ENTITY2);
    batchWriter.add(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddAutoId() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(PARTIAL_ENTITY_1.toPb())
        .addInsertAutoId(PARTIAL_ENTITY_2.toPb())
        .build();
    batchWriter.add(ENTITY1, PARTIAL_ENTITY_1);
    batchWriter.add(PARTIAL_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAutoIdWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(PARTIAL_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(PARTIAL_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.add(PARTIAL_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
index f3f7a5f62b09..d6c0679ef90a 100644
--- a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.protobuf.InvalidProtocolBufferException;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final PartialKey PARTIAL_KEY = PartialKey.builder("ds1", "k1").build();
  private static final PartialEntity PARTIAL_ENTITY = PartialEntity.builder(PARTIAL_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override
    public BaseEntity build() {

      return new BaseEntity(ImmutableSortedMap.copyOf(properties)) {
        @Override
        protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {

        }

        @Override
        protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
          return null;
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.protobuf.InvalidProtocolBufferException;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey PARTIAL_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final PartialEntity PARTIAL_ENTITY = PartialEntity.builder(PARTIAL_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override
    public BaseEntity build() {

      return new BaseEntity(ImmutableSortedMap.copyOf(properties)) {
        @Override
        protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {

        }

        @Override
        protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
          return null;
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 71433d7f178f..4226178bbc14 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -34,7 +34,7 @@ public class DatastoreHelperTest {
   @Test
   public void testFetch() throws Exception {
     DatastoreService datastoreService = createStrictMock(DatastoreService.class);
-    PartialKey pKey1 = PartialKey.builder("ds", "k").build();
+    IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
     Key key1 = Key.builder(pKey1, 1).build();
     Key key2 = Key.builder(pKey1, "a").build();
     Entity entity1 = Entity.builder(key1).build();
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index e16572d683ad..0218e17a012f 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -51,8 +51,8 @@ public class DatastoreServiceTest {
   private static final NullValue NULL_VALUE = NullValue.of();
   private static final StringValue STR_VALUE = StringValue.of("str");
   private static final BooleanValue BOOL_VALUE = BooleanValue.builder(false).indexed(false).build();
-  private static final PartialKey PARTIAL_KEY1 = PartialKey.builder(DATASET, KIND1).build();
-  private static final PartialKey PARTIAL_KEY2 = PartialKey.builder(DATASET, KIND2).build();
+  private static final IncompleteKey PARTIAL_KEY1 = IncompleteKey.builder(DATASET, KIND1).build();
+  private static final IncompleteKey PARTIAL_KEY2 = IncompleteKey.builder(DATASET, KIND2).build();
   private static final Key KEY1 = Key.builder(PARTIAL_KEY1, "name").build();
   private static final Key KEY2 = Key.builder(KEY1, KIND2, 1).build();
   private static final Key KEY3 = Key.builder(KEY2).name("bla").build();
@@ -70,7 +70,7 @@ public class DatastoreServiceTest {
   private static final PartialEntity PARTIAL_ENTITY2 = PartialEntity.builder(PARTIAL_ENTITY1)
       .remove("str").set("bool", true).set("list", LIST_VALUE1.get()).build();
   private static final PartialEntity PARTIAL_ENTITY3 = PartialEntity.builder(PARTIAL_ENTITY1)
-      .key(PartialKey.builder(DATASET, KIND3).build()).build();
+      .key(IncompleteKey.builder(DATASET, KIND3).build()).build();
   private static final Entity ENTITY1 = Entity.builder(KEY1)
       .set("str", STR_VALUE)
       .set("date", DATE_TIME_VALUE)
@@ -428,7 +428,7 @@ public void testRunStructuredQuery() {
   @Test
   public void testAllocateId() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
-    PartialKey pk1 = keyFactory.newKey();
+    IncompleteKey pk1 = keyFactory.newKey();
     Key key1 = keyFactory.allocateId();
     assertEquals(key1.dataset(), pk1.dataset());
     assertEquals(key1.namespace(), pk1.namespace());
@@ -450,16 +450,16 @@ public void testAllocateId() {
   @Test
   public void testAllocateIdArray() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
-    PartialKey partialKey1 = keyFactory.newKey();
-    PartialKey partialKey2 = keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
+    IncompleteKey incompleteKey1 = keyFactory.newKey();
+    IncompleteKey incompleteKey2 = keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
     Key key3 = keyFactory.newKey("name");
     Key key4 = keyFactory.newKey(1);
     List result =
-        datastore.allocateId(partialKey1, partialKey2, key3, key4, partialKey1, key3);
+        datastore.allocateId(incompleteKey1, incompleteKey2, key3, key4, incompleteKey1, key3);
     assertEquals(6, result.size());
-    assertEquals(Key.builder(partialKey1, result.get(0).id()).build(), result.get(0));
-    assertEquals(Key.builder(partialKey1, result.get(4).id()).build(), result.get(4));
-    assertEquals(Key.builder(partialKey2, result.get(1).id()).build(), result.get(1));
+    assertEquals(Key.builder(incompleteKey1, result.get(0).id()).build(), result.get(0));
+    assertEquals(Key.builder(incompleteKey1, result.get(4).id()).build(), result.get(4));
+    assertEquals(Key.builder(incompleteKey2, result.get(1).id()).build(), result.get(1));
     assertEquals(Key.builder(key3).id(result.get(2).id()).build(), result.get(2));
     assertEquals(Key.builder(key3).id(result.get(5).id()).build(), result.get(5));
     assertEquals(Key.builder(key4).id(result.get(3).id()).build(), result.get(3));
@@ -622,7 +622,7 @@ public void testDelete() {
   public void testKeyFactory() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     assertEquals(PARTIAL_KEY1, keyFactory.newKey());
-    assertEquals(PartialKey.builder(PARTIAL_KEY1).kind(KIND2).build(),
+    assertEquals(IncompleteKey.builder(PARTIAL_KEY1).kind(KIND2).build(),
         new KeyFactory(datastore).kind(KIND2).newKey());
     assertEquals(KEY1, keyFactory.newKey("name"));
     assertEquals(Key.builder(KEY1).id(2).build(), keyFactory.newKey(2));
diff --git a/src/test/java/com/google/gcloud/datastore/EntityTest.java b/src/test/java/com/google/gcloud/datastore/EntityTest.java
index a60ad3b18b80..c6569aa342ca 100644
--- a/src/test/java/com/google/gcloud/datastore/EntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/EntityTest.java
@@ -25,7 +25,7 @@ public class EntityTest {
 
   private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
   private static final Key KEY2 = Key.builder("ds1", "k2", 1).build();
-  private static final PartialKey PARTIAL_KEY = PartialKey.builder("ds1", "k2").build();
+  private static final IncompleteKey PARTIAL_KEY = IncompleteKey.builder("ds1", "k2").build();
   private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
   private static final PartialEntity PARTIAL_ENTITY =
       PartialEntity.builder(PARTIAL_KEY).set("a", "b").build();
diff --git a/src/test/java/com/google/gcloud/datastore/PartialKeyTest.java b/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
similarity index 80%
rename from src/test/java/com/google/gcloud/datastore/PartialKeyTest.java
rename to src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
index 36cf395ec842..2650ae9fe183 100644
--- a/src/test/java/com/google/gcloud/datastore/PartialKeyTest.java
+++ b/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
@@ -21,23 +21,23 @@
 
 import org.junit.Test;
 
-public class PartialKeyTest {
+public class IncompleteKeyTest {
 
   @Test
   public void testBuilders() throws Exception {
-    PartialKey pk1 = PartialKey.builder("ds", "kind1").build();
+    IncompleteKey pk1 = IncompleteKey.builder("ds", "kind1").build();
     assertEquals("ds", pk1.dataset());
     assertEquals("kind1", pk1.kind());
     assertTrue(pk1.ancestors().isEmpty());
 
     Key parent = Key.builder("ds", "kind2", 10).build();
-    PartialKey pk2 = PartialKey.builder(parent, "kind3").build();
+    IncompleteKey pk2 = IncompleteKey.builder(parent, "kind3").build();
     assertEquals("ds", pk2.dataset());
     assertEquals("kind3", pk2.kind());
     assertEquals(parent.path(), pk2.ancestors());
 
-    assertEquals(pk2, PartialKey.builder(pk2).build());
-    PartialKey pk3 = PartialKey.builder(pk2).kind("kind4").build();
+    assertEquals(pk2, IncompleteKey.builder(pk2).build());
+    IncompleteKey pk3 = IncompleteKey.builder(pk2).kind("kind4").build();
     assertEquals("ds", pk3.dataset());
     assertEquals("kind4", pk3.kind());
     assertEquals(parent.path(), pk3.ancestors());
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index c58253b3189e..97529d26305b 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -56,13 +56,13 @@ public void testNewKey() throws Exception {
   }
 
   @Test
-  public void testNewPartialKey() throws Exception {
-    PartialKey key = keyFactory.newKey();
-    verifyPartialKey(key, null);
+  public void testNewIncompletelKey() throws Exception {
+    IncompleteKey key = keyFactory.newKey();
+    verifyIncompleteKey(key, null);
     PathElement p1 = PathElement.of("k1", "n");
     PathElement p2 = PathElement.of("k2", 10);
     key = keyFactory.namespace("ns").ancestors(p1, p2).newKey();
-    verifyPartialKey(key, "ns", p1, p2);
+    verifyIncompleteKey(key, "ns", p1, p2);
   }
 
   @Test(expected = NullPointerException.class)
@@ -72,15 +72,15 @@ public void testNewPartialWithNoKind() {
 
   private void verifyKey(Key key, String name, String namespace, PathElement... ancestors) {
     assertEquals(name, key.name());
-    verifyPartialKey(key, namespace, ancestors);
+    verifyIncompleteKey(key, namespace, ancestors);
   }
 
   private void verifyKey(Key key, Long id, String namespace, PathElement... ancestors) {
     assertEquals(id, key.id());
-    verifyPartialKey(key, namespace, ancestors);
+    verifyIncompleteKey(key, namespace, ancestors);
   }
 
-  private void verifyPartialKey(PartialKey key, String namespace, PathElement... ancestors) {
+  private void verifyIncompleteKey(IncompleteKey key, String namespace, PathElement... ancestors) {
     assertEquals("k", key.kind());
     assertEquals(DATASET, key.dataset());
     assertEquals(namespace, key.namespace());
@@ -93,7 +93,7 @@ private void verifyPartialKey(PartialKey key, String namespace, PathElement... a
 
   @Test
   public void testAllocateId() throws Exception {
-    PartialKey pk = keyFactory.newKey();
+    IncompleteKey pk = keyFactory.newKey();
     Key key = keyFactory.newKey(1);
     DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder();
     requestPb.addKey(pk.toPb());
diff --git a/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java b/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
index 1e5f677c024f..9406f45736ad 100644
--- a/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
@@ -25,7 +25,7 @@
 public class PartialEntityTest {
 
   private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
-  private static final PartialKey PARTIAL_KEY = PartialKey.builder("ds1", "k2").build();
+  private static final IncompleteKey PARTIAL_KEY = IncompleteKey.builder("ds1", "k2").build();
   private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
   private static final PartialEntity PARTIAL_ENTITY =
       PartialEntity.builder(PARTIAL_KEY).set("a", "b").build();
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 89b7b01aadb3..06aee8bd4602 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -39,11 +39,11 @@
 
 public class SerializationTest {
 
-  private static final PartialKey INCOMPLETE_KEY1 =
-      PartialKey.builder("ds", "k").ancestors(PathElement.of("p", 1)).build();
+  private static final IncompleteKey INCOMPLETE_KEY1 =
+      IncompleteKey.builder("ds", "k").ancestors(PathElement.of("p", 1)).build();
   private static final Key KEY1 = Key.builder("ds", "k", "n").build();
-  private static final PartialKey INCOMPLETE_KEY2 =
-      PartialKey.builder(KEY1, "v").ancestors(PathElement.of("p", 1)).build();
+  private static final IncompleteKey INCOMPLETE_KEY2 =
+      IncompleteKey.builder(KEY1, "v").ancestors(PathElement.of("p", 1)).build();
   private static final Key KEY2 = Key.builder(KEY1, "v", 2).build();
   private static final DateTime DATE_TIME1 = DateTime.now();
   private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world"));

From 12bf706d67efc4690c00024cd54defff15c0d94e Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 11 Mar 2015 14:14:51 -0700
Subject: [PATCH 130/732] datastore entity change (remove partialentity)

---
 pom.xml                                       |   1 +
 .../datastore/BaseDatastoreBatchWriter.java   |  98 +++++++---
 .../google/gcloud/datastore/BaseEntity.java   |   6 +-
 .../datastore/DatastoreBatchWriter.java       |  33 ++--
 .../gcloud/datastore/DatastoreHelper.java     |  24 ++-
 .../gcloud/datastore/DatastoreReader.java     |   6 +-
 .../gcloud/datastore/DatastoreService.java    |  35 +---
 .../datastore/DatastoreServiceImpl.java       |  70 +++-----
 .../gcloud/datastore/DatastoreWriter.java     |  30 +++-
 .../com/google/gcloud/datastore/Entity.java   | 100 ++++++++---
 .../google/gcloud/datastore/EntityValue.java  |  20 +--
 .../com/google/gcloud/datastore/GqlQuery.java |   4 +-
 .../gcloud/datastore/IncompleteKey.java       |   3 +-
 .../gcloud/datastore/PartialEntity.java       | 134 --------------
 .../gcloud/datastore/ProjectionEntity.java    |   4 +
 .../com/google/gcloud/datastore/Query.java    |  31 ++--
 .../gcloud/datastore/StructuredQuery.java     |   2 +-
 .../google/gcloud/datastore/Transaction.java  |   6 +-
 .../gcloud/datastore/TransactionImpl.java     |   9 +-
 .../com/google/gcloud/datastore/Value.java    |   5 +-
 .../google/gcloud/datastore/package-info.java |   2 +-
 .../BaseDatastoreBatchWriterTest.java         |   2 +-
 .../gcloud/datastore/BaseEntityTest.java      |   2 +-
 .../gcloud/datastore/BlobValueTest.java       |   2 +
 .../gcloud/datastore/BooleanValueTest.java    |   2 +
 .../gcloud/datastore/DatastoreHelperTest.java |  52 +++++-
 .../datastore/DatastoreServiceTest.java       | 167 ++++++++----------
 .../gcloud/datastore/DateTimeValueTest.java   |   2 +
 .../gcloud/datastore/DoubleValueTest.java     |   2 +
 .../google/gcloud/datastore/EntityTest.java   |  31 ++--
 .../gcloud/datastore/EntityValueTest.java     |   2 +
 .../gcloud/datastore/KeyFactoryTest.java      |   4 +-
 .../google/gcloud/datastore/KeyValueTest.java |   2 +
 .../gcloud/datastore/ListValueTest.java       |   2 +
 .../gcloud/datastore/LongValueTest.java       |   2 +
 .../gcloud/datastore/NullValueTest.java       |   2 +
 .../gcloud/datastore/PartialEntityTest.java   |  62 -------
 .../google/gcloud/datastore/RawValueTest.java |   2 +
 .../gcloud/datastore/SerializationTest.java   |  14 +-
 .../gcloud/datastore/StringValueTest.java     |   2 +
 40 files changed, 468 insertions(+), 511 deletions(-)
 delete mode 100644 src/main/java/com/google/gcloud/datastore/PartialEntity.java
 delete mode 100644 src/test/java/com/google/gcloud/datastore/PartialEntityTest.java

diff --git a/pom.xml b/pom.xml
index 71e63e3ac9a4..9a112e25c12f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -196,6 +196,7 @@
           1.7
           1.7
           UTF-8
+          -Xlint:unchecked
         
       
       
diff --git a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
index 311965e54eac..adc086e47176 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
@@ -17,13 +17,10 @@
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 /**
  * Base class for DatastoreBatchWriter.
@@ -32,7 +29,7 @@ public abstract class BaseDatastoreBatchWriter implements DatastoreBatchWriter {
 
   private final String name;
   private final Map toAdd = new LinkedHashMap<>();
-  private final List toAddAutoId = new LinkedList<>();
+  private final List> toAddAutoId = new LinkedList<>();
   private final Map toUpdate = new LinkedHashMap<>();
   private final Map toPut = new LinkedHashMap<>();
   private final Set toDelete = new LinkedHashSet<>();
@@ -42,39 +39,79 @@ protected BaseDatastoreBatchWriter(String name) {
     this.name = name;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
-  public void add(Entity... entities) {
+  public final void addWithDeferredIdAllocation(Entity... entities) {
     validateActive();
-    for (Entity entity : entities) {
-      Key key = entity.key();
-      if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) {
-        throw newInvalidRequest("Entity with the key %s was already added or updated in this %s",
-            entity.key(), name);
-      }
-      if (toDelete.remove(key)) {
-        toPut.put(key, entity);
+    for (Entity entity : entities) {
+      IncompleteKey key = entity.key();
+      Preconditions.checkArgument(key != null, "Entity must have a key");
+      if (key instanceof Key) {
+        addInternal((Entity) entity);
       } else {
-        toAdd.put(key, entity);
+        toAddAutoId.add((Entity) entity);
       }
     }
   }
 
+  private void addInternal(Entity entity) {
+    Key key = entity.key();
+    if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) {
+      throw newInvalidRequest("Entity with the key %s was already added or updated in this %s",
+          entity.key(), name);
+    }
+    if (toDelete.remove(key)) {
+      toPut.put(key, entity);
+    } else {
+      toAdd.put(key, entity);
+    }
+  }
+
   @Override
-  public void add(PartialEntity... entities) {
+  public final Entity add(Entity entity) {
+    return DatastoreHelper.add(this, entity);
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public final List> add(Entity... entities) {
     validateActive();
-    for (PartialEntity entity : entities) {
-      if (entity instanceof Entity) {
-        add((Entity) entity);
+    ArrayList incompleteKeys = new ArrayList<>();
+    for (Entity entity : entities) {
+      IncompleteKey key = entity.key();
+      Preconditions.checkArgument(key != null, "Entity must have a key");
+      if (key instanceof Key) {
+        addInternal((Entity) entity);
+      } else {
+        incompleteKeys.add(key);
+      }
+    }
+    Iterator allocated;
+    if (!incompleteKeys.isEmpty()) {
+      IncompleteKey[] toAllocate = incompleteKeys.toArray(new IncompleteKey[incompleteKeys.size()]);
+      allocated = datastore().allocateId(toAllocate).iterator();
+    } else {
+      allocated = Collections.emptyIterator();
+    }
+    List> answer = Lists.newArrayListWithExpectedSize(entities.length);
+    for (Entity entity : entities) {
+      IncompleteKey key = entity.key();
+      if (key instanceof Key) {
+        answer.add((Entity) entity);
       } else {
-        toAddAutoId.add(entity);
+        Entity entityWithAllocatedId = Entity.builder(allocated.next(), entity).build();
+        addInternal(entityWithAllocatedId);
+        answer.add(entityWithAllocatedId);
       }
     }
+    return answer;
   }
 
+  @SafeVarargs
   @Override
-  public void update(Entity... entities) {
+  public final void update(Entity... entities) {
     validateActive();
-    for (Entity entity : entities) {
+    for (Entity entity : entities) {
       Key key = entity.key();
       if (toDelete.contains(key)) {
         throw newInvalidRequest("Entity with the key %s was already deleted in this %s",
@@ -88,10 +125,11 @@ public void update(Entity... entities) {
     }
   }
 
+  @SafeVarargs
   @Override
-  public void put(Entity... entities) {
+  public final void put(Entity... entities) {
     validateActive();
-    for (Entity entity : entities) {
+    for (Entity entity : entities) {
       Key key = entity.key();
       toAdd.remove(key);
       toUpdate.remove(key);
@@ -101,7 +139,7 @@ public void put(Entity... entities) {
   }
 
   @Override
-  public void delete(Key... keys) {
+  public final void delete(Key... keys) {
     validateActive();
     for (Key key : keys) {
       toAdd.remove(key);
@@ -124,7 +162,7 @@ protected Map toAdd() {
     return toAdd;
   }
 
-  protected List toAddAutoId() {
+  protected List> toAddAutoId() {
     return toAddAutoId;
   }
 
@@ -156,7 +194,7 @@ protected DatastoreServiceException newInvalidRequest(String msg, Object... para
 
   protected DatastoreV1.Mutation.Builder toMutationPb() {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    for (PartialEntity entity : toAddAutoId()) {
+    for (Entity entity : toAddAutoId()) {
       mutationPb.addInsertAutoId(entity.toPb());
     }
     for (Entity entity : toAdd().values()) {
@@ -173,4 +211,6 @@ protected DatastoreV1.Mutation.Builder toMutationPb() {
     }
     return mutationPb;
   }
+
+  protected abstract DatastoreService datastore();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 2a132a235bf8..cea40428f33b 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -112,7 +112,7 @@ public B set(String name, Key value) {
       return self();
     }
 
-    public B set(String name, PartialEntity value) {
+    public B set(String name, Entity value) {
       properties.put(name, of(value));
       return self();
     }
@@ -200,8 +200,8 @@ public Key getKey(String name) {
   }
 
   @SuppressWarnings("unchecked")
-  public  T getEntity(String name) {
-    return (T) ((Value) getValue(name)).get();
+  public  Entity getEntity(String name) {
+    return ((Value>) getValue(name)).get();
   }
 
   @SuppressWarnings("unchecked")
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
index dce06fb56304..59b57c0ba0d3 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
@@ -16,31 +16,36 @@
 
 package com.google.gcloud.datastore;
 
+import java.util.List;
+
 /**
  * An interface to represent a batch of write operations.
  * All write operation for a batch writer will be applied to the Datastore in one RPC call.
  */
 interface DatastoreBatchWriter extends DatastoreWriter {
 
-  /**
-   * {@inheritDoc}
-   * This operation will be converted to {@link #put} operation for entities that were already
-   *     marked for deletion in this writer.
-   * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity already added
-   *     to this writer or if not active
-   */
-  @Override
-  void add(Entity... entity);
-
   /**
    * Datastore add operation.
-   * This method will automatically allocate id for any entity with an incomplete key.
+   * This method will also allocate id for any entity with an incomplete key.
+   * As oppose to {@link #add(Entity)}, this method will defer any necessary id allocation
+   * to submit time.
    *
    * @throws IllegalArgumentException if any of the given entities is missing a key
    * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity with a
    *     complete key was already added to this writer or if not active
    */
-  void add(PartialEntity... entity);
+  void addWithDeferredIdAllocation(Entity... entity);
+
+  /**
+   * {@inheritDoc}
+   * For entities with complete keys that were marked for deletion in this writer the operation
+   * will be changed to {@link #put}.
+   * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity with the
+   *     same complete key was already added to this writer, if writer is not active or
+   *     if id allocation for an entity with an incomplete key failed.
+   */
+  @Override
+  List> add(Entity... entity);
 
   /**
    * {@inheritDoc}
@@ -50,7 +55,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    *     deletion in this writer or if not active
    */
   @Override
-  void update(Entity... entity);
+  void update(Entity... entity);
 
   /**
    * {@inheritDoc}
@@ -67,7 +72,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    * @throws com.google.gcloud.datastore.DatastoreServiceException if not active
    */
   @Override
-  void put(Entity... entity);
+  void put(Entity... entity);
 
   /**
    * Returns {@code true} if still active (write operations were not sent to the Datastore).
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index f650f4e44b3a..ac7671760d68 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.datastore;
 
+import com.google.common.collect.Iterators;
 import com.google.common.collect.Maps;
 
 import java.util.ArrayList;
@@ -32,18 +33,31 @@ class DatastoreHelper {
   private DatastoreHelper() {
   }
 
+
+  static Key allocateId(DatastoreService service, IncompleteKey key) {
+    return service.allocateId(new IncompleteKey[]{key}).get(0);
+  }
+
+  static Entity get(DatastoreReader reader, Key key) {
+    return Iterators.getNext(reader.get(new Key[]{key}), null);
+  }
+
+  static Entity add(DatastoreWriter writer, Entity entity) {
+    return writer.add(new Entity[] {entity}).get(0);
+  }
+
   /**
    * Returns a list with a value for each given key (ordered by input).
    * A {@code null} would be returned for non-existing keys.
    */
-  static List fetch(DatastoreReader reader, Key... keys) {
-    Iterator entities = reader.get(keys);
-    Map map = Maps.newHashMapWithExpectedSize(keys.length);
+  static List> fetch(DatastoreReader reader, Key... keys) {
+    Iterator> entities = reader.get(keys);
+    Map> map = Maps.newHashMapWithExpectedSize(keys.length);
     while (entities.hasNext()) {
-      Entity entity = entities.next();
+      Entity entity = entities.next();
       map.put(entity.key(), entity);
     }
-    List list = new ArrayList<>(keys.length);
+    List> list = new ArrayList<>(keys.length);
     for (Key key : keys) {
       // this will include nulls for non-existing keys
       list.add(map.get(key));
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index b1dc0a70d392..2bbd2764dc35 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -29,7 +29,7 @@ public interface DatastoreReader {
    *
    * @throws DatastoreServiceException upon failure.
    */
-  Entity get(Key key);
+  Entity get(Key key);
 
   /**
    * Returns an {@link Entity} for each given {@link Key} that exists in the Datastore.
@@ -41,14 +41,14 @@ public interface DatastoreReader {
    * @throws DatastoreServiceException upon failure.
    * @see #get(Key)
    */
-  Iterator get(Key... key);
+  Iterator> get(Key... key);
 
   /**
    * Returns a list with a value for each given key (ordered by input).
    * A {@code null} would be returned for non-existing keys.
    * When possible prefer using {@link #get(Key...)} which does not eagerly loads the results.
    */
-  List fetch(Key... keys);
+  List> fetch(Key... keys);
 
   /**
    * Submit a {@link Query} and returns its result.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index 7c678f6b3199..464d2308bbe3 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -81,50 +81,19 @@ interface TransactionCallable {
    */
   List allocateId(IncompleteKey... key);
 
-  /**
-   * Datastore add operation.
-   * This method will automatically allocate an id if necessary.
-   *
-   * @param entity the entity to add
-   * @return an {@code Entity} with the same properties and a key that is either newly allocated
-   *     or the same one if was already complete
-   * @throws DatastoreServiceException upon failure
-   * @throws IllegalArgumentException if the given entity is missing a key
-   */
-  Entity add(PartialEntity entity);
-
-  /**
-   * Datastore add operation.
-   * This method will automatically allocate id for any entity with incomplete key.
-   *
-   * @return a list of {@code Entity} ordered by input with the same properties and a key that is
-   *     either newly allocated or the same one if was already complete
-   * @throws DatastoreServiceException upon failure
-   * @throws IllegalArgumentException if any of the given entities is missing a key
-   * @see #add(PartialEntity)
-   */
-  List add(PartialEntity... entity);
-
-  /**
-   * {@inheritDoc}
-   * @throws DatastoreServiceException upon failure
-   */
-  @Override
-  void add(Entity... entity);
-
   /**
    * {@inheritDoc}
    * @throws DatastoreServiceException upon failure
    */
   @Override
-  void update(Entity... entity);
+  void update(Entity... entity);
 
   /**
    * {@inheritDoc}
    * @throws DatastoreServiceException upon failure
    */
   @Override
-  void put(Entity... entity);
+  void put(Entity... entity);
 
   /**
    * {@inheritDoc}
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 00b2689c9ecc..6fadadf2f831 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -118,7 +118,7 @@ DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestP
 
   @Override
   public Key allocateId(IncompleteKey key) {
-    return allocateId(new IncompleteKey[]{key}).get(0);
+    return DatastoreHelper.allocateId(this, key);
   }
 
   @Override
@@ -162,23 +162,22 @@ private IncompleteKey trimNameOrId(IncompleteKey key) {
   }
 
   @Override
-  public Entity add(PartialEntity entity) {
-    return add(new PartialEntity[] {entity}).get(0);
+  public Entity add(Entity entity) {
+    return DatastoreHelper.add(this, entity);
   }
 
+  @SuppressWarnings("unchecked")
   @Override
-  public List add(PartialEntity... entities) {
+  public List> add(Entity... entities) {
     if (entities.length == 0) {
       return Collections.emptyList();
     }
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
     Map completeEntities = new LinkedHashMap<>();
-    for (PartialEntity entity : entities) {
-      Entity completeEntity = null;
-      if (entity instanceof  Entity) {
-        completeEntity = (Entity) entity;
-      } else if (entity.key() instanceof Key) {
-        completeEntity = Entity.builder((Key) entity.key(), entity).build();
+    for (Entity entity : entities) {
+      Entity completeEntity = null;
+      if (entity.key() instanceof Key) {
+        completeEntity = (Entity) entity;
       }
       if (completeEntity != null) {
         if (completeEntities.put(completeEntity.key(), completeEntity) != null) {
@@ -194,10 +193,10 @@ public List add(PartialEntity... entities) {
     DatastoreV1.CommitResponse commitResponse = commitMutation(mutationPb);
     Iterator allocatedKeys =
         commitResponse.getMutationResult().getInsertAutoIdKeyList().iterator();
-    ImmutableList.Builder responseBuilder = ImmutableList.builder();
-    for (PartialEntity entity : entities) {
+    ImmutableList.Builder> responseBuilder = ImmutableList.builder();
+    for (Entity entity : entities) {
       IncompleteKey key = entity.key();
-      Entity completeEntity = completeEntities.get(key);
+      Entity completeEntity = completeEntities.get(key);
       if (completeEntity != null) {
         responseBuilder.add(completeEntity);
       } else {
@@ -208,21 +207,21 @@ public List add(PartialEntity... entities) {
   }
 
   @Override
-  public Entity get(Key key) {
-    return Iterators.getNext(get(new Key[]{key}), null);
+  public Entity get(Key key) {
+    return DatastoreHelper.get(this, key);
   }
 
   @Override
-  public Iterator get(Key... keys) {
+  public Iterator> get(Key... keys) {
     return get(null, keys);
   }
 
   @Override
-  public List fetch(Key... keys) {
+  public List> fetch(Key... keys) {
     return DatastoreHelper.fetch(this, keys);
   }
 
-  Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) {
+  Iterator> get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) {
     if (keys.length == 0) {
       return Collections.emptyIterator();
     }
@@ -236,7 +235,7 @@ Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) {
     return new ResultsIterator(requestPb);
   }
 
-  final class ResultsIterator extends AbstractIterator {
+  final class ResultsIterator extends AbstractIterator> {
 
     private final DatastoreV1.LookupRequest.Builder requestPb;
     Iterator iter;
@@ -255,8 +254,9 @@ private void loadResults() {
       }
     }
 
+    @SuppressWarnings("unchecked")
     @Override
-    protected Entity computeNext() {
+    protected Entity computeNext() {
       if (iter.hasNext()) {
         return Entity.fromPb(iter.next().getEntity());
       }
@@ -282,28 +282,13 @@ DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) {
     }
   }
 
+  @SafeVarargs
   @Override
-    public void add(Entity... entities) {
+  public final void update(Entity... entities) {
     if (entities.length > 0) {
       DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-      Set keys = new LinkedHashSet<>();
-      for (Entity entity : entities) {
-        if (!keys.add(entity.key())) {
-          throw DatastoreServiceException.throwInvalidRequest(
-              "Duplicate entity with the key %s", entity.key());
-        }
-        mutationPb.addInsert(entity.toPb());
-      }
-      commitMutation(mutationPb);
-    }
-  }
-
-  @Override
-  public void update(Entity... entities) {
-    if (entities.length > 0) {
-      DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-      Map dedupEntities = new LinkedHashMap<>();
-      for (Entity entity : entities) {
+      Map> dedupEntities = new LinkedHashMap<>();
+      for (Entity entity : entities) {
         dedupEntities.put(entity.key(), entity);
       }
       for (Entity entity : dedupEntities.values()) {
@@ -313,12 +298,13 @@ public void update(Entity... entities) {
     }
   }
 
+  @SafeVarargs
   @Override
-  public void put(Entity... entities) {
+  public final void put(Entity... entities) {
     if (entities.length > 0) {
       DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-      Map dedupEntities = new LinkedHashMap<>();
-      for (Entity entity : entities) {
+      Map> dedupEntities = new LinkedHashMap<>();
+      for (Entity entity : entities) {
         dedupEntities.put(entity.key(), entity);
       }
       for (Entity e : dedupEntities.values()) {
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
index 69e911c65e14..255ab8fa8a04 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
@@ -16,28 +16,48 @@
 
 package com.google.gcloud.datastore;
 
+import java.util.List;
+
 /**
  * An interface to represent Google Cloud Datastore write operations.
  */
 public interface DatastoreWriter {
 
   /**
-   * A Datastore add operation.
-   * The operation will fail if an entity with the same key already exists.
+   * Datastore add operation.
+   * This method will automatically allocate an id if necessary.
+   *
+   * @param entity the entity to add
+   * @return an {@code Entity} with the same properties and a key that is either newly allocated
+   *     or the same one if key is already complete
+   * @throws DatastoreServiceException upon failure
+   * @throws IllegalArgumentException if the given entity is missing a key
+   */
+  Entity add(Entity entity);
+
+  /**
+   * Datastore add operation.
+   * This method will automatically allocate id for any entity with an incomplete key.
+   *
+   * @return a list of {@code Entity} ordered by input with the same properties and a key that
+   *     is either newly allocated or the same one if was already complete
+   * @throws DatastoreServiceException upon failure
+   * @throws IllegalArgumentException if any of the given entities is missing a key
+   * @see #add(Entity)
    */
-  void add(Entity... entity);
+  List> add(Entity... entity);
 
   /**
    * A Datastore update operation.
    * The operation will fail if an entity with the same key does not already exist.
    */
-  void update(Entity... entity);
+  void update(Entity... entity);
 
   /**
    * A Datastore put (a.k.a upsert) operation.
    * The operation will add or modify the entities.
    */
-  void put(Entity... entity);
+  void put(Entity... entity);
 
   /**
    * A datastore delete operation.
diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java
index 95d96ac8030a..62a3141ce368 100644
--- a/src/main/java/com/google/gcloud/datastore/Entity.java
+++ b/src/main/java/com/google/gcloud/datastore/Entity.java
@@ -19,10 +19,11 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSortedMap;
 import com.google.protobuf.InvalidProtocolBufferException;
 
+import java.util.Objects;
+
 /**
  * An entity is the Google Cloud Datastore persistent data object.
  * An entity holds one or more properties, represented by a name (as {@link String})
@@ -32,49 +33,85 @@
  *
  * @see Google Cloud Datastore Entities, Properties, and Keys
  */
-public final class Entity extends PartialEntity {
+public final class Entity extends BaseEntity {
 
   private static final long serialVersionUID = 432961565733066915L;
 
-  public static final class Builder extends BaseEntity.Builder {
+  private final K key;
+
+  public static final class Builder
+      extends BaseEntity.Builder> {
+
+    private K key;
 
-    private Key key;
+    private Builder() {
+    }
 
-    private Builder(Key key) {
+    private Builder(K key) {
       this.key = checkNotNull(key);
     }
 
-    private Builder(Entity entity) {
+    private Builder(Entity entity) {
       super(entity);
       key = entity.key();
     }
 
-    private Builder(Key key, BaseEntity entity) {
+    private Builder(K key, BaseEntity entity) {
       super(entity);
-      this.key = key;
+      this.key = checkNotNull(key);
     }
 
-    public Builder key(Key key) {
+    public Builder key(K key) {
       this.key = checkNotNull(key);
       return this;
     }
 
     @Override
-    public Entity build() {
-      return new Entity(key, ImmutableSortedMap.copyOf(properties));
+    public Entity build() {
+      return new Entity<>(key, ImmutableSortedMap.copyOf(properties));
+    }
+  }
+
+  Entity(K key, ImmutableSortedMap> properties) {
+    super(properties);
+    this.key = key;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(key, properties());
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+    if (!(obj instanceof Entity)) {
+      return false;
+    }
+    Entity other = (Entity) obj;
+    return Objects.equals(key, other.key)
+        && Objects.equals(properties(), other.properties());
+  }
+
+  @Override
+  protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {
+    if (key != null) {
+      entityPb.setKey(key.toPb());
     }
   }
 
-  Entity(Key key, ImmutableSortedMap> properties) {
-    super(key, properties);
+  public boolean hasKey() {
+    return key != null;
   }
 
   /**
-   * Returns the entity's key (never null).
+   * Returns the entity's key.
+   * Entity will always have a key where as Entity may or may not have a key.
    */
-  @Override
-  public Key key() {
-    return (Key) super.key();
+  public K key() {
+    return key;
   }
 
   @Override
@@ -83,20 +120,31 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   }
 
   static Entity fromPb(DatastoreV1.Entity entityPb) {
-    PartialEntity entity = PartialEntity.fromPb(entityPb);
-    Preconditions.checkState(entity instanceof Entity, "Entity is not complete");
-    return (Entity) entity;
+    ImmutableSortedMap.Builder> properties =
+        ImmutableSortedMap.naturalOrder();
+    for (DatastoreV1.Property property : entityPb.getPropertyList()) {
+      properties.put(property.getName(), Value.fromPb(property.getValue()));
+    }
+    IncompleteKey key = null;
+    if (entityPb.hasKey()) {
+      key = IncompleteKey.fromPb(entityPb.getKey());
+    }
+    return new Entity<>(key, properties.build());
+  }
+
+  public static Builder builder() {
+    return new Builder<>();
   }
 
-  public static Builder builder(Key key) {
-    return new Builder(key);
+  public static  Builder builder(K key) {
+    return new Builder<>(key);
   }
 
-  public static Builder builder(Entity copyFrom) {
-    return new Builder(copyFrom);
+  public static  Builder builder(Entity copyFrom) {
+    return new Builder<>(copyFrom);
   }
 
-  public static Builder builder(Key key, PartialEntity copyFrom) {
-    return new Builder(key, copyFrom);
+  public static  Builder builder(K key, Entity copyFrom) {
+    return new Builder<>(key, copyFrom);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java
index 071375e78d16..68478c8d97cb 100644
--- a/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -21,12 +21,12 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Preconditions;
 
-public class EntityValue extends Value {
+public class EntityValue extends Value> {
 
   private static final long serialVersionUID = -5461475706792576395L;
 
-  static final BaseMarshaller MARSHALLER =
-      new BaseMarshaller() {
+  static final BaseMarshaller, EntityValue, Builder> MARSHALLER =
+      new BaseMarshaller, EntityValue, Builder>() {
 
         private static final long serialVersionUID = 2355075086076070931L;
 
@@ -36,13 +36,13 @@ public int getProtoFieldId() {
         }
 
         @Override
-        public Builder newBuilder(PartialEntity value) {
+        public Builder newBuilder(Entity value) {
           return builder(value);
         }
 
         @Override
-        protected PartialEntity getValue(DatastoreV1.Value from) {
-          return PartialEntity.fromPb(from.getEntityValue());
+        protected Entity getValue(DatastoreV1.Value from) {
+          return Entity.fromPb(from.getEntityValue());
         }
 
         @Override
@@ -51,7 +51,7 @@ protected void setValue(EntityValue from, DatastoreV1.Value.Builder to) {
         }
       };
 
-  public static final class Builder extends Value.BaseBuilder {
+  public static final class Builder extends Value.BaseBuilder, EntityValue, Builder> {
 
     private Builder() {
       super(Type.ENTITY);
@@ -70,7 +70,7 @@ public EntityValue build() {
     }
   }
 
-  public EntityValue(PartialEntity entity) {
+  public EntityValue(Entity entity) {
     this(builder(entity));
   }
 
@@ -83,11 +83,11 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
-  public static EntityValue of(PartialEntity entity) {
+  public static EntityValue of(Entity entity) {
     return new EntityValue(entity);
   }
 
-  public static Builder builder(PartialEntity entity) {
+  public static Builder builder(Entity entity) {
     return new Builder().set(entity).indexed(false);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 3260a309ea6a..932812fb8d5a 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -226,7 +226,7 @@ public Builder setBinding(String name, Key... value) {
       return this;
     }
 
-    public Builder setBinding(String name, PartialEntity... value) {
+    public Builder setBinding(String name, Entity... value) {
       namedBindings.put(name, toBinding(name, EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
@@ -271,7 +271,7 @@ public Builder addBinding(Key... value) {
       return this;
     }
 
-    public Builder addBinding(PartialEntity... value) {
+    public Builder addBinding(Entity... value) {
       positionalBindings.add(toBinding(EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
diff --git a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
index 1da007f2092b..6a8852f83d75 100644
--- a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
+++ b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
@@ -24,8 +24,7 @@
 import java.util.List;
 
 /**
- * A partial key (without a name or id).
- * Could be used as metadata for {@link PartialEntity}.
+ * An incomplete key (without a name or id).
  * This class is immutable.
  */
 public class IncompleteKey extends BaseKey {
diff --git a/src/main/java/com/google/gcloud/datastore/PartialEntity.java b/src/main/java/com/google/gcloud/datastore/PartialEntity.java
deleted file mode 100644
index b8cb27f64abe..000000000000
--- a/src/main/java/com/google/gcloud/datastore/PartialEntity.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.datastore;
-
-import com.google.api.services.datastore.DatastoreV1;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-import java.util.Objects;
-
-/**
- * A partial entity holds one or more properties, represented by a name (as {@link String})
- * and a value (as {@link Value}).
- * For a list of possible values see {@link Value.Type}.
- * A partial entity also can be associated with a key (partial or full).
- * This class is immutable.
- */
-public class PartialEntity extends BaseEntity {
-
-  private static final long serialVersionUID = 6492561268709192891L;
-
-  private final transient IncompleteKey key;
-
-  public static class Builder extends BaseEntity.Builder {
-
-    private IncompleteKey key;
-
-    private Builder() {
-    }
-
-    private Builder(PartialEntity entity) {
-      super(entity);
-      key = entity.key();
-    }
-
-    public Builder key(IncompleteKey key) {
-      this.key = key;
-      return this;
-    }
-
-    @Override
-    public PartialEntity build() {
-      return new PartialEntity(key, ImmutableSortedMap.copyOf(properties));
-    }
-  }
-
-  protected PartialEntity(IncompleteKey key, ImmutableSortedMap> properties) {
-    super(properties);
-    this.key = key;
-  }
-
-  public boolean hasKey() {
-    return key != null;
-  }
-
-  /**
-   * Returns the key for this entity or {@code null} if it does not have one.
-   */
-  public IncompleteKey key() {
-    return key;
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(key, properties());
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-    if (!(obj instanceof PartialEntity)) {
-      return false;
-    }
-    PartialEntity other = (PartialEntity) obj;
-    return Objects.equals(key, other.key)
-        && Objects.equals(properties(), other.properties());
-  }
-
-  @Override
-  protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {
-    if (key != null) {
-      entityPb.setKey(key.toPb());
-    }
-  }
-
-  @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    return fromPb(DatastoreV1.Entity.parseFrom(bytesPb));
-  }
-
-  static PartialEntity fromPb(DatastoreV1.Entity entityPb) {
-    ImmutableSortedMap.Builder> properties =
-        ImmutableSortedMap.naturalOrder();
-    for (DatastoreV1.Property property : entityPb.getPropertyList()) {
-      properties.put(property.getName(), Value.fromPb(property.getValue()));
-    }
-    IncompleteKey incompleteKey = null;
-    if (entityPb.hasKey()) {
-      incompleteKey = IncompleteKey.fromPb(entityPb.getKey());
-      if (incompleteKey instanceof Key) {
-        return new Entity((Key) incompleteKey, properties.build());
-      }
-    }
-    return new PartialEntity(incompleteKey, properties.build());
-  }
-
-  public static Builder builder() {
-    return new Builder();
-  }
-
-  public static Builder builder(IncompleteKey key) {
-    return new Builder().key(key);
-  }
-
-  public static Builder builder(PartialEntity copyFrom) {
-    return new Builder(copyFrom);
-  }
-}
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
index 17d260332eb6..1aa95b37110c 100644
--- a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -137,6 +137,10 @@ public static Builder builder(ProjectionEntity copyFrom) {
     return new Builder(copyFrom);
   }
 
+  static Builder builder() {
+    return new Builder();
+  }
+
   @Override
   protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {
     if (key != null) {
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 2dd3b231e6ee..5540748b0e4d 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -38,6 +38,10 @@
 public abstract class Query extends Serializable {
 
   private static final long serialVersionUID = -2748141759901313101L;
+  private static final Object OBJECT_INSTANCE = new Object();
+  private static final Key KEY_INSTANCE = Key.builder("dummy", "dummy", "dummy").build();
+  private static final Entity ENTITY_INSTANCE = Entity.builder(KEY_INSTANCE).build();
+  private static final ProjectionEntity PROJECTION_ENTITY = ProjectionEntity.builder().build();
 
   private final Type type;
   private final String namespace;
@@ -54,7 +58,7 @@ public abstract static class Type implements java.io.Serializable {
     private static final Map>
         PB_TO_INSTANCE = Maps.newEnumMap(DatastoreV1.EntityResult.ResultType.class);
 
-    static final Type UNKNOWN = new Type(null, Object.class) {
+    static final Type UNKNOWN = new Type(null, OBJECT_INSTANCE) {
 
       private static final long serialVersionUID = 1602329532153860907L;
 
@@ -69,18 +73,20 @@ public abstract static class Type implements java.io.Serializable {
       }
     };
 
-    public static final Type FULL =
-        new Type(DatastoreV1.EntityResult.ResultType.FULL, Entity.class) {
+    public static final Type> FULL =
+        new Type>(DatastoreV1.EntityResult.ResultType.FULL, ENTITY_INSTANCE) {
 
       private static final long serialVersionUID = 7712959777507168274L;
 
-      @Override protected Entity convert(DatastoreV1.Entity entityPb) {
+      @SuppressWarnings("unchecked")
+      @Override
+      protected Entity convert(DatastoreV1.Entity entityPb) {
         return Entity.fromPb(entityPb);
       }
     };
 
     public static final Type KEY_ONLY =
-        new Type(DatastoreV1.EntityResult.ResultType.KEY_ONLY, Key.class) {
+        new Type(DatastoreV1.EntityResult.ResultType.KEY_ONLY, KEY_INSTANCE) {
 
       private static final long serialVersionUID = -8514289244104446252L;
 
@@ -90,7 +96,7 @@ public abstract static class Type implements java.io.Serializable {
     };
 
     public static final Type PROJECTION = new Type(
-        DatastoreV1.EntityResult.ResultType.PROJECTION, ProjectionEntity.class) {
+        DatastoreV1.EntityResult.ResultType.PROJECTION, PROJECTION_ENTITY) {
 
           private static final long serialVersionUID = -7591409419690650246L;
 
@@ -102,15 +108,16 @@ public abstract static class Type implements java.io.Serializable {
     private final Class resultClass;
     private final DatastoreV1.EntityResult.ResultType resultType;
 
-    private Type(DatastoreV1.EntityResult.ResultType typePb, Class resultClass) {
-      this.resultType = typePb;
-      this.resultClass = checkNotNull(resultClass);
-      if (typePb != null) {
-        PB_TO_INSTANCE.put(typePb, this);
+    @SuppressWarnings("unchecked")
+    private Type(DatastoreV1.EntityResult.ResultType resultType, V resultObject) {
+      this.resultType = resultType;
+      this.resultClass = (Class) checkNotNull(resultObject).getClass();
+      if (resultType != null) {
+        PB_TO_INSTANCE.put(resultType, this);
       }
     }
 
-    public Class resultClass() {
+    public Class resultClass() {
       return resultClass;
     }
 
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 61c83d734da4..d68e5ad7a592 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -992,7 +992,7 @@ private static StructuredQuery fromPb(Type type, String namespace,
     return builder.namespace(namespace).mergeFrom(queryPb).build();
   }
 
-  public static Builder builder() {
+  public static Builder> builder() {
     return new Builder<>(Type.FULL);
   }
 
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index d2ff472aa2a9..bd2866da142b 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -65,7 +65,7 @@ interface Response {
    * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
-  Entity get(Key key);
+  Entity get(Key key);
 
   /**
    * {@inheritDoc}
@@ -76,7 +76,7 @@ interface Response {
    * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
-  Iterator get(Key... key);
+  Iterator> get(Key... key);
 
   /**
    * {@inheritDoc}
@@ -86,7 +86,7 @@ interface Response {
    *
    * @throws DatastoreServiceException upon failure or if no longer active
    */
-  List fetch(Key... keys);
+  List> fetch(Key... keys);
 
   /**
    * {@inheritDoc}
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 99a415d0d3aa..df2fab4134e8 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -18,7 +18,6 @@
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.gcloud.datastore.TransactionOption.ForceWrites;
 import com.google.gcloud.datastore.TransactionOption.IsolationLevel;
@@ -71,12 +70,12 @@ public List generatedKeys() {
   }
 
   @Override
-  public Entity get(Key key) {
-    return Iterators.getNext(get(new Key[] {key}), null);
+  public Entity get(Key key) {
+    return DatastoreHelper.get(this, key);
   }
 
   @Override
-  public Iterator get(Key... keys) {
+  public Iterator> get(Key... keys) {
     validateActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
@@ -84,7 +83,7 @@ public Iterator get(Key... keys) {
   }
 
   @Override
-  public List fetch(Key... keys) {
+  public List> fetch(Key... keys) {
     validateActive();
     return DatastoreHelper.fetch(this, keys);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index 300321bbefa4..79266f71816c 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -63,7 +63,7 @@ public enum Type {
     STRING(StringValue.MARSHALLER),
 
     /**
-     * Represents an entity ({@link PartialEntity} or {@link Entity}) value.
+     * Represents an entity value.
      */
     ENTITY(EntityValue.MARSHALLER),
 
@@ -165,6 +165,7 @@ abstract static class BaseMarshaller, B extends Builder ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final Entity INCOMPLETE_ENTITY_1 =
      Entity.builder(INCOMPLETE_KEY).build();
  private static final Entity INCOMPLETE_ENTITY_2 =
      Entity.builder(INCOMPLETE_KEY).set("name", "dan").build();

  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    private final DatastoreService datastore;

    protected DatastoreBatchWriter() {
      super("test");
      datastore = EasyMock.createMock(DatastoreService.class);
      IncompleteKey[] expected = {INCOMPLETE_KEY, INCOMPLETE_KEY};
      List result = ImmutableList.of(KEY2, KEY3);
      expect(datastore.allocateId(expected)).andReturn(result).times(0, 1);
      replay(datastore);
    }

    @Override
    protected DatastoreService datastore() {
      return datastore;
    }

    void finish() {
      verify(datastore);
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @After
  public void tearDown() {
    batchWriter.finish();
  }

  @Test
  public void testAdd() throws Exception {
    Entity entity2 =
        Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(entity2.toPb())
        .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb())
        .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb())
        .build();
    List> entities = batchWriter
        .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2);
    assertEquals(pb, batchWriter.toMutationPb().build());
    assertEquals(ENTITY1, entities.get(0));
    assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1));
    assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2));
    assertEquals(entity2, entities.get(3));
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddWithDeferredAllocation() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb())
        .build();
    batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWithDeferredAllocationWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
index d6c0679ef90a..dba278c1857d 100644
--- a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.protobuf.InvalidProtocolBufferException;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey PARTIAL_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final PartialEntity PARTIAL_ENTITY = PartialEntity.builder(PARTIAL_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override
    public BaseEntity build() {

      return new BaseEntity(ImmutableSortedMap.copyOf(properties)) {
        @Override
        protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {

        }

        @Override
        protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
          return null;
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.protobuf.InvalidProtocolBufferException;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final Entity PARTIAL_ENTITY = Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override
    public BaseEntity build() {

      return new BaseEntity(ImmutableSortedMap.copyOf(properties)) {
        @Override
        protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {

        }

        @Override
        protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
          return null;
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/BlobValueTest.java b/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
index 08709b846048..56ef03ac278e 100644
--- a/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
@@ -32,6 +32,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     BlobValue value = BlobValue.of(CONTENT);
@@ -40,6 +41,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     BlobValue.Builder builder = BlobValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java b/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
index 178ed1707152..fb3e62e6e27d 100644
--- a/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
@@ -30,6 +30,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     BooleanValue value = BooleanValue.of(false);
@@ -38,6 +39,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     BooleanValue.Builder builder = BooleanValue.builder(true);
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 4226178bbc14..14c9a8733499 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -19,29 +19,71 @@
 import static junit.framework.TestCase.assertTrue;
 import static junit.framework.TestCase.fail;
 import static org.easymock.EasyMock.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
+import static org.junit.Assert.*;
 
 import com.google.common.collect.Iterators;
 import com.google.gcloud.datastore.DatastoreService.TransactionCallable;
 import org.easymock.EasyMock;
 import org.junit.Test;
 
+import java.util.Collections;
 import java.util.List;
 
 public class DatastoreHelperTest {
 
+  @Test
+  public void testAllocateId() throws Exception {
+    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
+    Key key1 = Key.builder(pKey1, 1).build();
+    expect(datastoreService.allocateId(new IncompleteKey[] {pKey1}))
+        .andReturn(Collections.singletonList(key1));
+    replay(datastoreService);
+    assertEquals(key1, DatastoreHelper.allocateId(datastoreService, pKey1));
+    verify(datastoreService);
+  }
+
+  @Test
+  public void testGet() throws Exception {
+    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
+    Key key1 = Key.builder(pKey1, 1).build();
+    Entity entity1 = Entity.builder(key1).build();
+    Key key2 = Key.builder(pKey1, 2).build();
+    expect(datastoreService.get(new Key[]{key1}))
+        .andReturn(Collections.singletonList(entity1).iterator());
+    expect(datastoreService.get(new Key[]{key2}))
+        .andReturn(Collections.>emptyIterator());
+    replay(datastoreService);
+    assertEquals(entity1, DatastoreHelper.get(datastoreService, key1));
+    assertNull(DatastoreHelper.get(datastoreService, key2));
+    verify(datastoreService);
+  }
+
+  @Test
+  public void testAdd() throws Exception {
+    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    IncompleteKey pKey = IncompleteKey.builder("ds", "k").build();
+    Key key = Key.builder(pKey, 1).build();
+    Entity entity = Entity.builder(key).build();
+    expect(datastoreService.add(new Entity[]{entity}))
+        .andReturn(Collections.singletonList(entity));
+    replay(datastoreService);
+    assertEquals(entity, DatastoreHelper.add(datastoreService, entity));
+    verify(datastoreService);
+  }
+
   @Test
   public void testFetch() throws Exception {
     DatastoreService datastoreService = createStrictMock(DatastoreService.class);
     IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
     Key key1 = Key.builder(pKey1, 1).build();
     Key key2 = Key.builder(pKey1, "a").build();
-    Entity entity1 = Entity.builder(key1).build();
-    Entity entity2 = Entity.builder(key2).build();
+    Entity entity1 = Entity.builder(key1).build();
+    Entity entity2 = Entity.builder(key2).build();
     expect(datastoreService.get(key1, key2)).andReturn(Iterators.forArray(entity1, entity2)).once();
     replay(datastoreService);
-    List values = DatastoreHelper.fetch(datastoreService, key1, key2);
+    List> values = DatastoreHelper.fetch(datastoreService, key1, key2);
     assertEquals(2, values.size());
     assertEquals(entity1, values.get(0));
     assertEquals(entity2, values.get(1));
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 0218e17a012f..84be43a46b11 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.datastore;
 
+import static junit.framework.TestCase.assertNotNull;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -24,6 +25,7 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.common.collect.Iterators;
 import com.google.gcloud.datastore.Query.Type;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
@@ -51,9 +53,9 @@ public class DatastoreServiceTest {
   private static final NullValue NULL_VALUE = NullValue.of();
   private static final StringValue STR_VALUE = StringValue.of("str");
   private static final BooleanValue BOOL_VALUE = BooleanValue.builder(false).indexed(false).build();
-  private static final IncompleteKey PARTIAL_KEY1 = IncompleteKey.builder(DATASET, KIND1).build();
-  private static final IncompleteKey PARTIAL_KEY2 = IncompleteKey.builder(DATASET, KIND2).build();
-  private static final Key KEY1 = Key.builder(PARTIAL_KEY1, "name").build();
+  private static final IncompleteKey INCOMPLETE_KEY1 = IncompleteKey.builder(DATASET, KIND1).build();
+  private static final IncompleteKey INCOMPLETE_KEY2 = IncompleteKey.builder(DATASET, KIND2).build();
+  private static final Key KEY1 = Key.builder(INCOMPLETE_KEY1, "name").build();
   private static final Key KEY2 = Key.builder(KEY1, KIND2, 1).build();
   private static final Key KEY3 = Key.builder(KEY2).name("bla").build();
   private static final Key KEY4 = Key.builder(KEY2).name("newName1").build();
@@ -65,22 +67,22 @@ public class DatastoreServiceTest {
       .build();
   private static final ListValue LIST_VALUE2 = ListValue.of(Collections.singletonList(KEY_VALUE));
   private static final DateTimeValue DATE_TIME_VALUE = new DateTimeValue(DateTime.now());
-  private static final PartialEntity PARTIAL_ENTITY1 = PartialEntity.builder(PARTIAL_KEY2)
+  private static final Entity PARTIAL_ENTITY1 = Entity.builder(INCOMPLETE_KEY2)
       .set("str", STR_VALUE).set("bool", BOOL_VALUE).set("list", LIST_VALUE1).build();
-  private static final PartialEntity PARTIAL_ENTITY2 = PartialEntity.builder(PARTIAL_ENTITY1)
+  private static final Entity PARTIAL_ENTITY2 = Entity.builder(PARTIAL_ENTITY1)
       .remove("str").set("bool", true).set("list", LIST_VALUE1.get()).build();
-  private static final PartialEntity PARTIAL_ENTITY3 = PartialEntity.builder(PARTIAL_ENTITY1)
+  private static final Entity PARTIAL_ENTITY3 = Entity.builder(PARTIAL_ENTITY1)
       .key(IncompleteKey.builder(DATASET, KIND3).build()).build();
-  private static final Entity ENTITY1 = Entity.builder(KEY1)
+  private static final Entity ENTITY1 = Entity.builder(KEY1)
       .set("str", STR_VALUE)
       .set("date", DATE_TIME_VALUE)
       .set("bool", BOOL_VALUE)
       .set("partial1", EntityValue.of(PARTIAL_ENTITY1))
       .set("list", LIST_VALUE2)
       .build();
-  private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
+  private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
       .set("name", "Dan").setNull("null").set("age", 20).build();
-  private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
+  private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
       .set("null", NULL_VALUE).set("partial1", PARTIAL_ENTITY2).set("partial2", ENTITY2).build();
 
   private DatastoreServiceOptions options;
@@ -102,8 +104,9 @@ public void setUp() throws IOException, InterruptedException {
         .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
     datastore = DatastoreServiceFactory.getDefault(options);
-    // Prepare data for testing
-    datastore.delete(KEY1, KEY2, KEY3, KEY4, KEY5);
+    StructuredQuery query = StructuredQuery.keyOnlyBuilder().build();
+    QueryResults result = datastore.run(query);
+    datastore.delete(Iterators.toArray(result, Key.class));
     datastore.add(ENTITY1, ENTITY2);
   }
 
@@ -131,7 +134,7 @@ public void testNewTransactionCommit() {
     transaction.delete(KEY1);
     transaction.commit();
 
-    List list = datastore.fetch(KEY1, KEY2, KEY3);
+    List> list = datastore.fetch(KEY1, KEY2, KEY3);
     assertNull(list.get(0));
     assertEquals(entity2, list.get(1));
     assertEquals(ENTITY3, list.get(2));
@@ -177,10 +180,10 @@ public void testTransactionWithRead() {
 
   @Test
   public void testTransactionWithQuery() {
-    Query query =
+    Query> query =
         StructuredQuery.builder().kind(KIND2).filter(PropertyFilter.hasAncestor(KEY2)).build();
     Transaction transaction = datastore.newTransaction();
-    QueryResults results = transaction.run(query);
+    QueryResults> results = transaction.run(query);
     assertEquals(ENTITY2, results.next());
     assertFalse(results.hasNext());
     transaction.add(ENTITY3);
@@ -221,7 +224,7 @@ public void testNewTransactionRollback() {
 
     verifyNotUsable(transaction);
 
-    List list = datastore.fetch(KEY1, KEY2, KEY3);
+    List> list = datastore.fetch(KEY1, KEY2, KEY3);
     assertEquals(ENTITY1, list.get(0));
     assertEquals(ENTITY2, list.get(1));
     assertNull(list.get(2));
@@ -261,27 +264,39 @@ private void verifyNotUsable(DatastoreWriter writer) {
   @Test
   public void testNewBatch() {
     Batch batch = datastore.newBatch();
-    Entity entity1 = Entity.builder(ENTITY1).clear().build();
-    Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
-    Entity entity4 = Entity.builder(KEY4).set("value", StringValue.of("value")).build();
-    Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
-
-    batch.add(entity4, entity5);
-    batch.add(PARTIAL_ENTITY3);
+    Entity entity1 = Entity.builder(ENTITY1).clear().build();
+    Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
+    Entity entity4 = Entity.builder(KEY4).set("value", StringValue.of("value")).build();
+    Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
+
+    List> entities = batch.add(entity4, PARTIAL_ENTITY2, entity5);
+    Entity entity6 = entities.get(1);
+    assertSame(entity4, entities.get(0));
+    assertEquals(PARTIAL_ENTITY2.properties(), entity6.properties());
+    assertEquals(PARTIAL_ENTITY2.key().dataset(), entity6.key().dataset());
+    assertEquals(PARTIAL_ENTITY2.key().namespace(), entity6.key().namespace());
+    assertEquals(PARTIAL_ENTITY2.key().ancestors(), entity6.key().ancestors());
+    assertEquals(PARTIAL_ENTITY2.key().kind(), entity6.key().kind());
+    assertEquals(PARTIAL_ENTITY2.key(), IncompleteKey.builder(entity6.key()).build());
+    assertNotEquals(PARTIAL_ENTITY2.key().path(), entity6.key().path());
+    assertNotEquals(PARTIAL_ENTITY2.key(), entity6.key());
+    assertSame(entity5, entities.get(2));
+    batch.addWithDeferredIdAllocation(PARTIAL_ENTITY3);
     batch.put(ENTITY3, entity1, entity2);
 
     Batch.Response response = batch.submit();
-    Iterator entities =
-        datastore.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
-    assertEquals(entity1, entities.next());
-    assertEquals(entity2, entities.next());
-    assertEquals(ENTITY3, entities.next());
-    assertEquals(entity4, entities.next());
-    assertEquals(entity5, entities.next());
-    assertFalse(entities.hasNext());
+    entities = datastore.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key(), entity6.key());
+    assertEquals(entity1, entities.get(0));
+    assertEquals(entity2, entities.get(1));
+    assertEquals(ENTITY3, entities.get(2));
+    assertEquals(entity4, entities.get(3));
+    assertEquals(entity5, entities.get(4));
+    assertEquals(entity6, entities.get(5));
+    assertEquals(6, entities.size());
     List generatedKeys = response.generatedKeys();
     assertEquals(1, generatedKeys.size());
-    assertEquals(PARTIAL_ENTITY1.properties(), datastore.get(generatedKeys.get(0)).properties());
+    assertEquals(PARTIAL_ENTITY3.properties(), datastore.get(generatedKeys.get(0)).properties());
+    assertEquals(PARTIAL_ENTITY3.key(), IncompleteKey.builder(generatedKeys.get(0)).build());
 
     try {
       batch.submit();
@@ -295,33 +310,27 @@ public void testNewBatch() {
     batch.delete(entity4.key(), entity5.key());
     batch.update(ENTITY1, ENTITY2, ENTITY3);
     batch.submit();
-    entities = datastore.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key()).iterator();
-    assertEquals(ENTITY1, entities.next());
-    assertEquals(ENTITY2, entities.next());
-    assertEquals(ENTITY3, entities.next());
-    assertNull(entities.next());
-    assertNull(entities.next());
-    assertFalse(entities.hasNext());
-
-    // TODO need to cover the following use-cases:
-    // delete after put/add/update
-    // put after delete/add/update
-    // update after delete/add/put
-    // add after delete/update/put
+    entities = datastore.fetch(KEY1, KEY2, KEY3, entity4.key(), entity5.key());
+    assertEquals(ENTITY1, entities.get(0));
+    assertEquals(ENTITY2, entities.get(1));
+    assertEquals(ENTITY3, entities.get(2));
+    assertNull(entities.get(3));
+    assertNull(entities.get(4));
+    assertEquals(5, entities.size());
   }
 
   @Test
   public void testRunGqlQueryNoCasting() {
-    Query query1 = GqlQuery.builder(Type.FULL, "select * from " + KIND1).build();
-    QueryResults results1 = datastore.run(query1);
+    Query> query1 = GqlQuery.builder(Type.FULL, "select * from " + KIND1).build();
+    QueryResults> results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
     datastore.put(ENTITY3);
-    Query query2 =  GqlQuery.builder(
+    Query query2 =  GqlQuery.builder(
         Type.FULL, "select * from " + KIND2 + " order by __key__").build();
-    QueryResults results2 = datastore.run(query2);
+    QueryResults results2 = datastore.run(query2);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY2, results2.next());
     assertTrue(results2.hasNext());
@@ -383,9 +392,9 @@ public void testRunGqlQueryWithCasting() {
 
   @Test
   public void testRunStructuredQuery() {
-    StructuredQuery query =
+    StructuredQuery> query =
         StructuredQuery.builder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
-    QueryResults results1 = datastore.run(query);
+    QueryResults> results1 = datastore.run(query);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
@@ -480,7 +489,7 @@ public void testGet() {
     assertEquals(LIST_VALUE2, value3);
     DateTimeValue value4 = entity.getValue("date");
     assertEquals(DATE_TIME_VALUE, value4);
-    PartialEntity value5 = entity.getEntity("partial1");
+    Entity value5 = entity.getEntity("partial1");
     assertEquals(PARTIAL_ENTITY1, value5);
     assertEquals(5, entity.names().size());
     assertFalse(entity.contains("bla"));
@@ -489,7 +498,7 @@ public void testGet() {
   @Test
   public void testGetArray() {
     datastore.put(ENTITY3);
-    Iterator result =
+    Iterator> result =
         datastore.fetch(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3).iterator();
     assertEquals(ENTITY1, result.next());
     assertNull(result.next());
@@ -499,7 +508,7 @@ public void testGetArray() {
     assertTrue(entity3.isNull("null"));
     assertFalse(entity3.getBoolean("bool"));
     assertEquals(LIST_VALUE2.get(), entity3.getList("list"));
-    PartialEntity partial1 = entity3.getEntity("partial1");
+    Entity partial1 = entity3.getEntity("partial1");
     Entity partial2 = entity3.getEntity("partial2");
     assertEquals(PARTIAL_ENTITY2, partial1);
     assertEquals(ENTITY2, partial2);
@@ -518,7 +527,7 @@ public void testGetArray() {
 
   @Test
   public void testAddEntity() {
-    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
+    List> keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -529,44 +538,22 @@ public void testAddEntity() {
     } catch (DatastoreServiceException expected) {
       // expected;
     }
-    datastore.add(ENTITY3);
+
+    List> entities = datastore.add(ENTITY3, PARTIAL_ENTITY1, PARTIAL_ENTITY2);
     assertEquals(ENTITY3, datastore.get(ENTITY3.key()));
+    assertEquals(ENTITY3, entities.get(0));
+    assertEquals(PARTIAL_ENTITY1.properties(), entities.get(1).properties());
+    assertEquals(PARTIAL_ENTITY1.key().ancestors(), entities.get(1).key().ancestors());
+    assertNotNull(datastore.get(entities.get(1).key()));
+    assertEquals(PARTIAL_ENTITY2.properties(), entities.get(2).properties());
+    assertEquals(PARTIAL_ENTITY2.key().ancestors(), entities.get(2).key().ancestors());
+    assertNotNull(datastore.get(entities.get(2).key()));
   }
 
-  @Test
-  public void testAddPartialEntity() {
-    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
-    assertEquals(ENTITY1, keys.get(0));
-    assertNull(keys.get(1));
-    assertEquals(2, keys.size());
-
-    try {
-      datastore.add(ENTITY1);
-      fail("Expecting a failure");
-    } catch (DatastoreServiceException expected) {
-      // expected;
-    }
-
-    PartialEntity pe = PartialEntity.builder(PARTIAL_ENTITY2).key(KEY5).build();
-    List response = datastore.add(PARTIAL_ENTITY3, ENTITY3, PARTIAL_ENTITY3, pe);
-    assertEquals(4, response.size());
-    assertEquals(PARTIAL_ENTITY3.properties(), response.get(0).properties());
-    assertEquals(PARTIAL_ENTITY3.properties(), datastore.get(response.get(0).key()).properties());
-    assertSame(ENTITY3, response.get(1));
-    assertEquals(ENTITY3, datastore.get(response.get(1).key()));
-    assertEquals(PARTIAL_ENTITY3.properties(), response.get(2).properties());
-    assertEquals(PARTIAL_ENTITY3.properties(), datastore.get(response.get(2).key()).properties());
-    assertEquals(pe.properties(), response.get(3).properties());
-    assertEquals(pe.key(), response.get(3).key());
-    assertEquals(pe.properties(), datastore.get(response.get(3).key()).properties());
-    assertEquals(pe.key(), datastore.get(response.get(3).key()).key());
-    assertEquals(pe, response.get(3));
-    assertEquals(datastore.get(response.get(3).key()), response.get(3));
-  }
 
   @Test
   public void testUpdate() {
-    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
+    List> keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -587,7 +574,7 @@ public void testUpdate() {
 
   @Test
   public void testPut() {
-    Iterator keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator> keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
@@ -605,7 +592,7 @@ public void testPut() {
 
   @Test
   public void testDelete() {
-    Iterator keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator> keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
@@ -621,8 +608,8 @@ public void testDelete() {
   @Test
   public void testKeyFactory() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
-    assertEquals(PARTIAL_KEY1, keyFactory.newKey());
-    assertEquals(IncompleteKey.builder(PARTIAL_KEY1).kind(KIND2).build(),
+    assertEquals(INCOMPLETE_KEY1, keyFactory.newKey());
+    assertEquals(IncompleteKey.builder(INCOMPLETE_KEY1).kind(KIND2).build(),
         new KeyFactory(datastore).kind(KIND2).newKey());
     assertEquals(KEY1, keyFactory.newKey("name"));
     assertEquals(Key.builder(KEY1).id(2).build(), keyFactory.newKey(2));
diff --git a/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java b/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
index 2c5436a3e25b..39e0d1ca3f01 100644
--- a/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
@@ -32,6 +32,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     DateTimeValue value = DateTimeValue.of(CONTENT);
@@ -40,6 +41,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     DateTimeValue.Builder builder = DateTimeValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java b/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
index c264c5f20586..012f9bbf4de9 100644
--- a/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
@@ -32,6 +32,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     DoubleValue value = DoubleValue.of(CONTENT);
@@ -40,6 +41,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     DoubleValue.Builder builder = DoubleValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/EntityTest.java b/src/test/java/com/google/gcloud/datastore/EntityTest.java
index c6569aa342ca..f4fc970e7ca9 100644
--- a/src/test/java/com/google/gcloud/datastore/EntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/EntityTest.java
@@ -16,8 +16,7 @@
 
 package com.google.gcloud.datastore;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.*;
 
 import org.junit.Test;
 
@@ -25,20 +24,28 @@ public class EntityTest {
 
   private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
   private static final Key KEY2 = Key.builder("ds1", "k2", 1).build();
-  private static final IncompleteKey PARTIAL_KEY = IncompleteKey.builder("ds1", "k2").build();
-  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
-  private static final PartialEntity PARTIAL_ENTITY =
-      PartialEntity.builder(PARTIAL_KEY).set("a", "b").build();
+  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k2").build();
+  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
+  private static final Entity INCOMPLETE_ENTITY =
+      Entity.builder(INCOMPLETE_KEY).set("a", "b").build();
 
   @Test
   public void testGetKey() throws Exception {
+    assertTrue(ENTITY.hasKey());
     assertEquals(KEY1, ENTITY.key());
     assertEquals("bar", ENTITY.getString("foo"));
   }
+  @Test
+  public void testNoKey() throws Exception {
+    Entity entity = Entity.builder().set("foo", "bar").build();
+    assertFalse(entity.hasKey());
+    assertNull(entity.key());
+    assertEquals("bar", entity.getString("foo"));
+  }
 
   @Test
   public void testCopyFrom() throws Exception {
-    Entity.Builder builder = Entity.builder(ENTITY);
+    Entity.Builder builder = Entity.builder(ENTITY);
     assertEquals(ENTITY, builder.build());
     Entity entity = builder.key(KEY2).build();
     assertNotEquals(ENTITY, entity);
@@ -47,10 +54,10 @@ public void testCopyFrom() throws Exception {
   }
 
   @Test
-  public void testCopyFromPartialEntity() throws Exception {
-    Entity.Builder builder = Entity.builder(KEY2, PARTIAL_ENTITY);
-    Entity entity = builder.build();
-    assertNotEquals(PARTIAL_ENTITY, entity);
-    assertEquals(PARTIAL_ENTITY.properties(), entity.properties());
+  public void testCopyFromIncompleteEntity() throws Exception {
+    Entity.Builder builder = Entity.builder(KEY2, INCOMPLETE_ENTITY);
+    Entity entity = builder.build();
+    assertNotEquals(INCOMPLETE_ENTITY, entity);
+    assertEquals(INCOMPLETE_ENTITY.properties(), entity.properties());
   }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
index 00f48fd9fc1a..dfacfb0b67a6 100644
--- a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
@@ -33,6 +33,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     EntityValue value = EntityValue.of(CONTENT);
@@ -47,6 +48,7 @@ public void testIndexedNotAllowed() {
     EntityValue.builder(CONTENT).indexed(true);
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     EntityValue.Builder builder = EntityValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index 97529d26305b..4de33197816b 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -56,7 +56,7 @@ public void testNewKey() throws Exception {
   }
 
   @Test
-  public void testNewIncompletelKey() throws Exception {
+  public void testNewIncompleteKey() throws Exception {
     IncompleteKey key = keyFactory.newKey();
     verifyIncompleteKey(key, null);
     PathElement p1 = PathElement.of("k1", "n");
@@ -66,7 +66,7 @@ public void testNewIncompletelKey() throws Exception {
   }
 
   @Test(expected = NullPointerException.class)
-  public void testNewPartialWithNoKind() {
+  public void testNewIncompleteWithNoKind() {
     new KeyFactory(keyFactory.datastore()).build();
   }
 
diff --git a/src/test/java/com/google/gcloud/datastore/KeyValueTest.java b/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
index ebbbf7101543..ea6b77ed97da 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
@@ -32,6 +32,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     KeyValue value = KeyValue.of(CONTENT);
@@ -40,6 +41,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     KeyValue.Builder builder = KeyValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/ListValueTest.java b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
index 4a0168fb2b1e..ae07f0c4fa51 100644
--- a/src/test/java/com/google/gcloud/datastore/ListValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
@@ -36,6 +36,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     ListValue value = ListValue.of(CONTENT);
@@ -49,6 +50,7 @@ public void testIndexedCannotBeSpecified() {
     ListValue.builder().indexed(false);
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     ListValue.Builder builder = ListValue.builder().set(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/LongValueTest.java b/src/test/java/com/google/gcloud/datastore/LongValueTest.java
index b0881ba7dd7a..d93dfaf331a8 100644
--- a/src/test/java/com/google/gcloud/datastore/LongValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/LongValueTest.java
@@ -32,6 +32,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     LongValue value = LongValue.of(CONTENT);
@@ -40,6 +41,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     LongValue.Builder builder = LongValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/NullValueTest.java b/src/test/java/com/google/gcloud/datastore/NullValueTest.java
index 24fc7ff2e0b1..15fe78dd1a4f 100644
--- a/src/test/java/com/google/gcloud/datastore/NullValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/NullValueTest.java
@@ -31,6 +31,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     NullValue value = NullValue.of();
@@ -39,6 +40,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     NullValue.Builder builder = NullValue.builder();
diff --git a/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java b/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
deleted file mode 100644
index 9406f45736ad..000000000000
--- a/src/test/java/com/google/gcloud/datastore/PartialEntityTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.datastore;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNull;
-
-import org.junit.Test;
-
-public class PartialEntityTest {
-
-  private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
-  private static final IncompleteKey PARTIAL_KEY = IncompleteKey.builder("ds1", "k2").build();
-  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
-  private static final PartialEntity PARTIAL_ENTITY =
-      PartialEntity.builder(PARTIAL_KEY).set("a", "b").build();
-
-  @Test
-  public void testGetKey() throws Exception {
-    assertEquals(PARTIAL_KEY, PARTIAL_ENTITY.key());
-    assertEquals("b", PARTIAL_ENTITY.getString("a"));
-  }
-
-  @Test
-  public void testNoKey() throws Exception {
-    PartialEntity entity = PartialEntity.builder().set("foo", "bar").build();
-    assertNull(entity.key());
-    assertEquals("bar", entity.getString("foo"));
-  }
-
-  @Test
-  public void testCopyFromPartialEntity() throws Exception {
-    PartialEntity entity = Entity.builder(PARTIAL_ENTITY).build();
-    assertEquals(PARTIAL_ENTITY, entity);
-    entity = Entity.builder(PARTIAL_ENTITY).key(KEY1).build();
-    assertNotEquals(PARTIAL_ENTITY, entity);
-    assertEquals(PARTIAL_ENTITY.properties(), entity.properties());
-    assertNotEquals(PARTIAL_ENTITY.key(), entity.key());
-    assertEquals(KEY1, entity.key());
-  }
-
-  @Test
-  public void testCopyFromEntity() throws Exception {
-    PartialEntity entity = Entity.builder(ENTITY).build();
-    assertEquals(ENTITY, entity);
-  }
-}
diff --git a/src/test/java/com/google/gcloud/datastore/RawValueTest.java b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
index 48cc88c20e63..26854ff9d22b 100644
--- a/src/test/java/com/google/gcloud/datastore/RawValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
@@ -34,6 +34,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     RawValue value = RawValue.of(CONTENT);
@@ -42,6 +43,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     RawValue.Builder builder = RawValue.builder(CONTENT);
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 06aee8bd4602..dd98b0c36a7d 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -55,13 +55,13 @@ public class SerializationTest {
       .addBinding(20)
       .namespace("ns1")
       .build();
-  private static final Query GQL2 =
+  private static final Query> GQL2 =
       GqlQuery.builder(Query.Type.FULL, "select * from kind1 where name = @name and age > @1")
       .setBinding("name", "name1")
       .addBinding(20)
       .namespace("ns1")
       .build();
-  private static final Query QUERY1 = StructuredQuery.builder().kind("kind1").build();
+  private static final Query> QUERY1 = StructuredQuery.builder().kind("kind1").build();
   private static final Query QUERY2 = StructuredQuery.keyOnlyBuilder()
       .kind("k")
       .filter(PropertyFilter.eq("p1", "hello"))
@@ -88,8 +88,8 @@ public class SerializationTest {
   private static final BlobValue BLOB_VALUE = BlobValue.of(BLOB1);
   private static final RawValue RAW_VALUE = RawValue.of(
       DatastoreV1.Value.newBuilder().setBlobKeyValue("blob-key").setMeaning(18).build());
-  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
-  private static final Entity ENTITY2 =
+  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
+  private static final Entity ENTITY2 =
       Entity.builder(KEY2).set("null", NullValue.of()).build();
   private static final Entity ENTITY3 = Entity.builder(KEY2)
       .set("p1", StringValue.builder("hi1").meaning(10).build())
@@ -97,9 +97,9 @@ public class SerializationTest {
       .set("p3", LongValue.builder(100).indexed(false).meaning(100).build())
       .set("blob", BLOB1)
       .build();
-  private static final PartialEntity EMBEDDED_ENTITY1 = ENTITY1;
-  private static final PartialEntity EMBEDDED_ENTITY2 = ENTITY2;
-  private static final PartialEntity EMBEDDED_ENTITY3 = PartialEntity.builder(INCOMPLETE_KEY1)
+  private static final Entity EMBEDDED_ENTITY1 = (Entity) ENTITY1;
+  private static final Entity EMBEDDED_ENTITY2 = (Entity) ENTITY2;
+  private static final Entity EMBEDDED_ENTITY3 = Entity.builder(INCOMPLETE_KEY1)
       .set("p1", STRING_VALUE)
       .set("p2", LongValue.builder(100).indexed(false).meaning(100).build())
       .build();
diff --git a/src/test/java/com/google/gcloud/datastore/StringValueTest.java b/src/test/java/com/google/gcloud/datastore/StringValueTest.java
index 42e83b6a4850..aa53cea22d3a 100644
--- a/src/test/java/com/google/gcloud/datastore/StringValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/StringValueTest.java
@@ -32,6 +32,7 @@ public void testToBuilder() throws Exception {
     assertEquals(value, value.toBuilder().build());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testOf() throws Exception {
     StringValue value = StringValue.of(CONTENT);
@@ -40,6 +41,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testBuilder() throws Exception {
     StringValue.Builder builder = StringValue.builder(CONTENT);

From 3dbb4859b71645980227d125e61699605bdff69e Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 11 Mar 2015 17:22:52 -0700
Subject: [PATCH 131/732] add FullQuery/FullQueryBuilder and move builder
 methods to Query

---
 .../com/google/gcloud/datastore/GqlQuery.java | 22 +----
 .../com/google/gcloud/datastore/Query.java    | 42 +++++++++
 .../gcloud/datastore/StructuredQuery.java     | 88 +++++++++++--------
 .../datastore/DatastoreServiceTest.java       | 48 +++++-----
 .../gcloud/datastore/SerializationTest.java   | 11 +--
 5 files changed, 130 insertions(+), 81 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 932812fb8d5a..cd70d35ac9f6 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -44,7 +44,8 @@
  *
  * 
When the type of the results is known the preferred usage would be:
  * 
{@code
- *   Query<Entity> query = GqlQuery.builder(Query.Type.FULL, "select * from kind").build();
+ *   Query<Entity> query =
+ *       Query.gqlQueryBuilder(Query.Type.FULL, "select * from kind").build();
  *   QueryResults<Entity> results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
@@ -54,7 +55,7 @@
  *
  * 
When the type of the results is unknown you can use this approach:
  * 
{@code
- *   Query<?> query = GqlQuery.builder("select __key__ from kind").build();
+ *   Query<?> query = Query.gqlQueryBuilder("select __key__ from kind").build();
  *   QueryResults<?> results = datastore.run(query);
  *   if (Key.class.isAssignableFrom(results.resultClass())) {
  *     QueryResults<Key> keys = (QueryResults<Key>) results;
@@ -416,21 +417,4 @@ private static  GqlQuery fromPb(Type type, String ns, DatastoreV1.GqlQu
     return builder.build();
   }
 
-  /**
-   * Returns a new GQL query builder.
-   *
-   * @see GQL Reference
-   */
-  public static GqlQuery.Builder builder(String gql) {
-    return builder(Type.UNKNOWN, gql);
-  }
-
-  /**
-   * Returns a new GQL query builder.
-   *
-   * @see GQL Reference
-   */
-  public static  GqlQuery.Builder builder(Type type, String gql) {
-    return new GqlQuery.Builder<>(type, gql);
-  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 5540748b0e4d..c5b8705ee5c7 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -22,6 +22,9 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
 import com.google.common.collect.Maps;
+import com.google.gcloud.datastore.StructuredQuery.FullQueryBuilder;
+import com.google.gcloud.datastore.StructuredQuery.KeyOnlyQueryBuilder;
+import com.google.gcloud.datastore.StructuredQuery.ProjectionQueryBuilder;
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 
@@ -190,4 +193,43 @@ protected abstract Object fromPb(Type type, String namespace, byte[] bytesPb)
   protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb);
 
   protected abstract Query nextQuery(DatastoreV1.QueryResultBatch responsePb);
+
+  /**
+   * Returns a new {@link GqlQuery} builder.
+   *
+   * @see GQL Reference
+   */
+  public static GqlQuery.Builder gqlQueryBuilder(String gql) {
+    return gqlQueryBuilder(Type.UNKNOWN, gql);
+  }
+
+  /**
+   * Returns a new {@link GqlQuery} builder.
+   *
+   * @see GQL Reference
+   */
+  public static  GqlQuery.Builder gqlQueryBuilder(Type type, String gql) {
+    return new GqlQuery.Builder<>(type, gql);
+  }
+
+  /**
+   * Returns a new {@link StructuredQuery} builder for full queries.
+   */
+  public static FullQueryBuilder fullQueryBuilder() {
+    return new FullQueryBuilder();
+  }
+
+  /**
+   * Returns a new {@link StructuredQuery} builder for key-only queries.
+   */
+  public static KeyOnlyQueryBuilder keyOnlyQueryBuilder() {
+    return new KeyOnlyQueryBuilder();
+  }
+
+  /**
+   * Returns a new {@link StructuredQuery} builder for projection queries.
+   */
+  public static ProjectionQueryBuilder projectionQueryBuilder() {
+    return new ProjectionQueryBuilder();
+  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index d68e5ad7a592..0eceeeffd6f8 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -46,19 +46,26 @@
  * 
A usage example:

  *
  * 
A simple query that returns all entities for a specific kind
- * 
 {@code
- *   StructuredQuery query = StructuredQuery.builder().kind(kind).build();
- *   QueryResults results = datastore.run(query);
+ * 
{@code
+ *   FullQuery query = Query.fullQueryBuilder().kind(kind).build();
+ *   QueryResults> results = datastore.run(query);
  *   while (results.hasNext()) {
- *     Entity entity = results.next();
+ *     Entity entity = results.next();
  *     ...
  *   }
  * } 

  *
+ * 
A simple key-only query of all entities for a specific kind
+ * 
{@code
+ *   KeyOnlyQuery keyOnlyQuery =  Query.keyOnlyQueryBuilder().kind(KIND1).build();
+ *   QueryResults results = datastore.run(keyOnlyQuery);
+ *   ...
+ * } 

+ *
  * 
A less trivial example of a projection query that returns the first 10 results
  * of "age" and "name" properties (sorted and grouped by "age") with an age greater than 18
- * 
 {@code
- *   StructuredQuery query = StructuredQuery.projectionBuilder()
+ * 
{@code
+ *   ProjectionQuery query = Query.projectionQueryBuilder()
  *       .kind(kind)
  *       .projection(Projection.property("age"), Projection.first("name"))
  *       .filter(PropertyFilter.gt("age", 18))
@@ -739,22 +746,34 @@ public StructuredQuery build() {
     }
   }
 
-  public static final class Builder extends BaseBuilder> {
+  static final class Builder extends BaseBuilder> {
 
-    public Builder(Type type) {
+    Builder(Type type) {
       super(type);
     }
   }
 
-  public static final class KeyOnlyBuilder extends BaseBuilder {
+  public static final class FullQueryBuilder extends BaseBuilder, FullQueryBuilder> {
 
-    public KeyOnlyBuilder() {
+    FullQueryBuilder() {
+      super(Type.FULL);
+    }
+
+    @Override
+    public FullQuery build() {
+      return new FullQuery(this);
+    }
+  }
+
+  public static final class KeyOnlyQueryBuilder extends BaseBuilder {
+
+    KeyOnlyQueryBuilder() {
       super(Type.KEY_ONLY);
       projection(Projection.property(KEY_PROPERTY_NAME));
     }
 
     @Override
-    protected KeyOnlyBuilder mergeFrom(DatastoreV1.Query queryPb) {
+    protected KeyOnlyQueryBuilder mergeFrom(DatastoreV1.Query queryPb) {
       super.mergeFrom(queryPb);
       projection(Projection.property(KEY_PROPERTY_NAME));
       clearGroupBy();
@@ -767,10 +786,10 @@ public KeyOnlyQuery build() {
     }
   }
 
-  public static final class ProjectionBuilder
-      extends BaseBuilder {
+  public static final class ProjectionQueryBuilder
+      extends BaseBuilder {
 
-    public ProjectionBuilder() {
+    ProjectionQueryBuilder() {
       super(Type.PROJECTION);
     }
 
@@ -780,41 +799,48 @@ public ProjectionQuery build() {
     }
 
     @Override
-    public ProjectionBuilder clearProjection() {
+    public ProjectionQueryBuilder clearProjection() {
       return super.clearProjection();
     }
 
     @Override
-    public ProjectionBuilder projection(Projection projection, Projection... others) {
+    public ProjectionQueryBuilder projection(Projection projection, Projection... others) {
       return super.projection(projection, others);
     }
 
     @Override
-    public ProjectionBuilder addProjection(Projection projection, Projection... others) {
+    public ProjectionQueryBuilder addProjection(Projection projection, Projection... others) {
       return super.addProjection(projection, others);
     }
 
     @Override
-    public ProjectionBuilder clearGroupBy() {
+    public ProjectionQueryBuilder clearGroupBy() {
       return super.clearGroupBy();
     }
 
     @Override
-    public ProjectionBuilder groupBy(String property, String... others) {
+    public ProjectionQueryBuilder groupBy(String property, String... others) {
       return super.groupBy(property, others);
     }
 
     @Override
-    public ProjectionBuilder addGroupBy(String property, String... others) {
+    public ProjectionQueryBuilder addGroupBy(String property, String... others) {
       return super.addGroupBy(property, others);
     }
   }
 
+  public static final class FullQuery extends StructuredQuery> {
+
+    FullQuery(BaseBuilder, ?> builder) {
+      super(builder);
+    }
+  }
+
   public static final class KeyOnlyQuery extends StructuredQuery {
 
     private static final long serialVersionUID = -7502917784216095473L;
 
-    KeyOnlyQuery(KeyOnlyBuilder builder) {
+    KeyOnlyQuery(KeyOnlyQueryBuilder builder) {
       super(builder);
     }
   }
@@ -823,7 +849,7 @@ public static final class ProjectionQuery extends StructuredQuery fromPb(Type type, String namespace,
       DatastoreV1.Query queryPb) {
     BaseBuilder builder;
     if (type.equals(Type.FULL)) {
-      builder = builder();
+      builder = new FullQueryBuilder();
     } else if (type.equals(Type.KEY_ONLY)) {
-      builder = keyOnlyBuilder();
+      builder = new KeyOnlyQueryBuilder();
     } else {
-      builder = projectionBuilder();
+      builder = new ProjectionQueryBuilder();
     }
     return builder.namespace(namespace).mergeFrom(queryPb).build();
   }
-
-  public static Builder> builder() {
-    return new Builder<>(Type.FULL);
-  }
-
-  public static KeyOnlyBuilder keyOnlyBuilder() {
-    return new KeyOnlyBuilder();
-  }
-
-  public static ProjectionBuilder projectionBuilder() {
-    return new ProjectionBuilder();
-  }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 84be43a46b11..0b8a1ea6d005 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -27,6 +27,7 @@
 
 import com.google.common.collect.Iterators;
 import com.google.gcloud.datastore.Query.Type;
+import com.google.gcloud.datastore.StructuredQuery.FullQuery;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
@@ -104,7 +105,7 @@ public void setUp() throws IOException, InterruptedException {
         .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
     datastore = DatastoreServiceFactory.getDefault(options);
-    StructuredQuery query = StructuredQuery.keyOnlyBuilder().build();
+    StructuredQuery query = Query.keyOnlyQueryBuilder().build();
     QueryResults result = datastore.run(query);
     datastore.delete(Iterators.toArray(result, Key.class));
     datastore.add(ENTITY1, ENTITY2);
@@ -180,8 +181,10 @@ public void testTransactionWithRead() {
 
   @Test
   public void testTransactionWithQuery() {
-    Query> query =
-        StructuredQuery.builder().kind(KIND2).filter(PropertyFilter.hasAncestor(KEY2)).build();
+    Query> query = Query.fullQueryBuilder()
+        .kind(KIND2)
+        .filter(PropertyFilter.hasAncestor(KEY2))
+        .build();
     Transaction transaction = datastore.newTransaction();
     QueryResults> results = transaction.run(query);
     assertEquals(ENTITY2, results.next());
@@ -321,14 +324,14 @@ public void testNewBatch() {
 
   @Test
   public void testRunGqlQueryNoCasting() {
-    Query> query1 = GqlQuery.builder(Type.FULL, "select * from " + KIND1).build();
+    Query> query1 = Query.gqlQueryBuilder(Type.FULL, "select * from " + KIND1).build();
     QueryResults> results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
     datastore.put(ENTITY3);
-    Query query2 =  GqlQuery.builder(
+    Query query2 =  Query.gqlQueryBuilder(
         Type.FULL, "select * from " + KIND2 + " order by __key__").build();
     QueryResults results2 = datastore.run(query2);
     assertTrue(results2.hasNext());
@@ -337,18 +340,18 @@ public void testRunGqlQueryNoCasting() {
     assertEquals(ENTITY3, results2.next());
     assertFalse(results2.hasNext());
 
-    query1 = GqlQuery.builder(Type.FULL, "select * from bla").build();
+    query1 = Query.gqlQueryBuilder(Type.FULL, "select * from bla").build();
     results1 = datastore.run(query1);
     assertFalse(results1.hasNext());
 
     Query keyOnlyQuery =
-        GqlQuery.builder(Type.KEY_ONLY, "select __key__ from " + KIND1).build();
+        Query.gqlQueryBuilder(Type.KEY_ONLY, "select __key__ from " + KIND1).build();
     QueryResults keyOnlyResults = datastore.run(keyOnlyQuery);
     assertTrue(keyOnlyResults.hasNext());
     assertEquals(KEY1, keyOnlyResults.next());
     assertFalse(keyOnlyResults.hasNext());
 
-    GqlQuery keyProjectionQuery = GqlQuery.builder(
+    GqlQuery keyProjectionQuery = Query.gqlQueryBuilder(
         Type.PROJECTION, "select __key__ from " + KIND1).build();
     QueryResults keyProjectionResult = datastore.run(keyProjectionQuery);
     assertTrue(keyProjectionResult.hasNext());
@@ -357,7 +360,7 @@ public void testRunGqlQueryNoCasting() {
     assertTrue(projectionEntity.properties().isEmpty());
     assertFalse(keyProjectionResult.hasNext());
 
-    GqlQuery projectionQuery = GqlQuery.builder(
+    GqlQuery projectionQuery = Query.gqlQueryBuilder(
         Type.PROJECTION, "select str, date from " + KIND1).build();
 
     QueryResults projectionResult = datastore.run(projectionQuery);
@@ -374,13 +377,14 @@ public void testRunGqlQueryNoCasting() {
   @Test
   public void testRunGqlQueryWithCasting() {
     @SuppressWarnings("unchecked")
-    Query query1 = (Query) GqlQuery.builder("select * from " + KIND1).build();
-    QueryResults results1 = datastore.run(query1);
+    Query> query1 =
+        (Query>) Query.gqlQueryBuilder("select * from " + KIND1).build();
+    QueryResults> results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
-    Query query2 = GqlQuery.builder("select * from " + KIND1).build();
+    Query query2 = Query.gqlQueryBuilder("select * from " + KIND1).build();
     QueryResults results2 = datastore.run(query2);
     assertSame(Entity.class, results2.resultClass());
     @SuppressWarnings("unchecked")
@@ -392,20 +396,21 @@ public void testRunGqlQueryWithCasting() {
 
   @Test
   public void testRunStructuredQuery() {
-    StructuredQuery> query =
-        StructuredQuery.builder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
+    FullQuery query =
+        Query.fullQueryBuilder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
     QueryResults> results1 = datastore.run(query);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
-    StructuredQuery keyOnlyQuery =  StructuredQuery.keyOnlyBuilder().kind(KIND1).build();
+    Query keyOnlyQuery =  Query.keyOnlyQueryBuilder().kind(KIND1).build();
     QueryResults results2 = datastore.run(keyOnlyQuery);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY1.key(), results2.next());
     assertFalse(results2.hasNext());
 
-    StructuredQuery keyOnlyProjectionQuery = StructuredQuery.projectionBuilder()
+    StructuredQuery keyOnlyProjectionQuery =
+        Query.projectionQueryBuilder()
         .kind(KIND1).projection(Projection.property("__key__")).build();
     QueryResults results3 = datastore.run(keyOnlyProjectionQuery);
     assertTrue(results3.hasNext());
@@ -414,7 +419,7 @@ public void testRunStructuredQuery() {
     assertTrue(projectionEntity.names().isEmpty());
     assertFalse(results2.hasNext());
 
-    StructuredQuery projectionQuery = StructuredQuery.projectionBuilder()
+    StructuredQuery projectionQuery = Query.projectionQueryBuilder()
         .kind(KIND2)
         .projection(Projection.property("age"), Projection.first("name"))
         .filter(PropertyFilter.gt("age", 18))
@@ -460,7 +465,8 @@ public void testAllocateId() {
   public void testAllocateIdArray() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     IncompleteKey incompleteKey1 = keyFactory.newKey();
-    IncompleteKey incompleteKey2 = keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
+    IncompleteKey incompleteKey2 =
+        keyFactory.kind(KIND2).ancestors(PathElement.of(KIND1, 10)).newKey();
     Key key3 = keyFactory.newKey("name");
     Key key4 = keyFactory.newKey(1);
     List result =
@@ -574,7 +580,8 @@ public void testUpdate() {
 
   @Test
   public void testPut() {
-    Iterator> keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator> keys =
+        datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
@@ -592,7 +599,8 @@ public void testPut() {
 
   @Test
   public void testDelete() {
-    Iterator> keys = datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
+    Iterator> keys =
+        datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
     assertNull(keys.next());
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index dd98b0c36a7d..63b2f1fc2cdd 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -50,23 +50,24 @@ public class SerializationTest {
   private static final Cursor CURSOR1 = Cursor.copyFrom(new byte[] {1,2});
   private static final Cursor CURSOR2 = Cursor.copyFrom(new byte[] {10});
   private static final Query GQL1 =
-      GqlQuery.builder("select * from kind1 where name = @name and age > @1")
+      Query.gqlQueryBuilder("select * from kind1 where name = @name and age > @1")
       .setBinding("name", "name1")
       .addBinding(20)
       .namespace("ns1")
       .build();
   private static final Query> GQL2 =
-      GqlQuery.builder(Query.Type.FULL, "select * from kind1 where name = @name and age > @1")
+      Query.gqlQueryBuilder(Query.Type.FULL, "select * from kind1 where name = @name and age > @1")
       .setBinding("name", "name1")
       .addBinding(20)
       .namespace("ns1")
       .build();
-  private static final Query> QUERY1 = StructuredQuery.builder().kind("kind1").build();
-  private static final Query QUERY2 = StructuredQuery.keyOnlyBuilder()
+  private static final Query> QUERY1 =
+      Query.fullQueryBuilder().kind("kind1").build();
+  private static final Query QUERY2 = Query.keyOnlyQueryBuilder()
       .kind("k")
       .filter(PropertyFilter.eq("p1", "hello"))
       .build();
-  private static final Query QUERY3 = StructuredQuery.projectionBuilder()
+  private static final Query QUERY3 = Query.projectionQueryBuilder()
       .kind("k")
       .namespace("ns1")
       .projection(Projection.property("p"))

From ae5f87f5fcf59b4672d463946b3f47432986658a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 13 Mar 2015 17:33:55 -0700
Subject: [PATCH 132/732] Rename Entity to FullEntity and provide an
 Entity as a shorthand for FullEntity (+ a  guaranty to have a key).

---
 .../datastore/BaseDatastoreBatchWriter.java   |  75 +++----
 .../google/gcloud/datastore/BaseEntity.java   | 188 ++++++++++++++++--
 .../datastore/DatastoreBatchWriter.java       |  10 +-
 .../gcloud/datastore/DatastoreHelper.java     |  16 +-
 .../gcloud/datastore/DatastoreReader.java     |   6 +-
 .../gcloud/datastore/DatastoreService.java    |   4 +-
 .../datastore/DatastoreServiceImpl.java       |  41 ++--
 .../gcloud/datastore/DatastoreWriter.java     |  12 +-
 .../com/google/gcloud/datastore/Entity.java   | 121 ++++-------
 .../google/gcloud/datastore/EntityValue.java  |  20 +-
 .../google/gcloud/datastore/FullEntity.java   |  76 +++++++
 .../com/google/gcloud/datastore/GqlQuery.java |   4 +-
 .../gcloud/datastore/ProjectionEntity.java    |  85 +-------
 .../com/google/gcloud/datastore/Query.java    |  24 +--
 .../gcloud/datastore/StructuredQuery.java     |  10 +-
 .../google/gcloud/datastore/Transaction.java  |   7 +-
 .../gcloud/datastore/TransactionImpl.java     |   6 +-
 .../google/gcloud/datastore/package-info.java |   2 +-
 .../BaseDatastoreBatchWriterTest.java         |   2 +-
 .../gcloud/datastore/BaseEntityTest.java      |   2 +-
 .../gcloud/datastore/DatastoreHelperTest.java |  12 +-
 .../datastore/DatastoreServiceTest.java       |  71 +++----
 .../google/gcloud/datastore/EntityTest.java   |  19 +-
 .../gcloud/datastore/FullEntityTest.java      |  72 +++++++
 .../datastore/ProjectionEntityTest.java       |   3 +-
 .../gcloud/datastore/SerializationTest.java   |  22 +-
 26 files changed, 547 insertions(+), 363 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/FullEntity.java
 create mode 100644 src/test/java/com/google/gcloud/datastore/FullEntityTest.java

diff --git a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
index adc086e47176..655b1a32125c 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
@@ -18,9 +18,17 @@
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 
-import java.util.*;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Base class for DatastoreBatchWriter.
@@ -28,10 +36,10 @@
 public abstract class BaseDatastoreBatchWriter implements DatastoreBatchWriter {
 
   private final String name;
-  private final Map toAdd = new LinkedHashMap<>();
-  private final List> toAddAutoId = new LinkedList<>();
-  private final Map toUpdate = new LinkedHashMap<>();
-  private final Map toPut = new LinkedHashMap<>();
+  private final Map> toAdd = new LinkedHashMap<>();
+  private final List> toAddAutoId = new LinkedList<>();
+  private final Map> toUpdate = new LinkedHashMap<>();
+  private final Map> toPut = new LinkedHashMap<>();
   private final Set toDelete = new LinkedHashSet<>();
   private boolean active = true;
 
@@ -41,20 +49,20 @@ protected BaseDatastoreBatchWriter(String name) {
 
   @SuppressWarnings("unchecked")
   @Override
-  public final void addWithDeferredIdAllocation(Entity... entities) {
+  public final void addWithDeferredIdAllocation(FullEntity... entities) {
     validateActive();
-    for (Entity entity : entities) {
+    for (FullEntity entity : entities) {
       IncompleteKey key = entity.key();
       Preconditions.checkArgument(key != null, "Entity must have a key");
       if (key instanceof Key) {
-        addInternal((Entity) entity);
+        addInternal((FullEntity) entity);
       } else {
-        toAddAutoId.add((Entity) entity);
+        toAddAutoId.add((FullEntity) entity);
       }
     }
   }
 
-  private void addInternal(Entity entity) {
+  private void addInternal(FullEntity entity) {
     Key key = entity.key();
     if (toAdd.containsKey(key) || toUpdate.containsKey(key) || toPut.containsKey(key)) {
       throw newInvalidRequest("Entity with the key %s was already added or updated in this %s",
@@ -68,38 +76,37 @@ private void addInternal(Entity entity) {
   }
 
   @Override
-  public final Entity add(Entity entity) {
+  public final Entity add(FullEntity entity) {
     return DatastoreHelper.add(this, entity);
   }
 
   @SuppressWarnings("unchecked")
   @Override
-  public final List> add(Entity... entities) {
+  public final List add(FullEntity... entities) {
     validateActive();
-    ArrayList incompleteKeys = new ArrayList<>();
-    for (Entity entity : entities) {
+    List incompleteKeys = Lists.newArrayListWithExpectedSize(entities.length);
+    for (FullEntity entity : entities) {
       IncompleteKey key = entity.key();
       Preconditions.checkArgument(key != null, "Entity must have a key");
       if (key instanceof Key) {
-        addInternal((Entity) entity);
+        addInternal((FullEntity) entity);
       } else {
         incompleteKeys.add(key);
       }
     }
     Iterator allocated;
     if (!incompleteKeys.isEmpty()) {
-      IncompleteKey[] toAllocate = incompleteKeys.toArray(new IncompleteKey[incompleteKeys.size()]);
+      IncompleteKey[] toAllocate = Iterables.toArray(incompleteKeys, IncompleteKey.class);
       allocated = datastore().allocateId(toAllocate).iterator();
     } else {
       allocated = Collections.emptyIterator();
     }
-    List> answer = Lists.newArrayListWithExpectedSize(entities.length);
-    for (Entity entity : entities) {
-      IncompleteKey key = entity.key();
-      if (key instanceof Key) {
-        answer.add((Entity) entity);
+    List answer = Lists.newArrayListWithExpectedSize(entities.length);
+    for (FullEntity entity : entities) {
+      if (entity.key() instanceof Key) {
+        answer.add(Entity.convert((FullEntity) entity));
       } else {
-        Entity entityWithAllocatedId = Entity.builder(allocated.next(), entity).build();
+        Entity entityWithAllocatedId = Entity.builder(allocated.next(), entity).build();
         addInternal(entityWithAllocatedId);
         answer.add(entityWithAllocatedId);
       }
@@ -109,9 +116,9 @@ public final List> add(Entity... entities) {
 
   @SafeVarargs
   @Override
-  public final void update(Entity... entities) {
+  public final void update(Entity... entities) {
     validateActive();
-    for (Entity entity : entities) {
+    for (Entity entity : entities) {
       Key key = entity.key();
       if (toDelete.contains(key)) {
         throw newInvalidRequest("Entity with the key %s was already deleted in this %s",
@@ -127,9 +134,9 @@ public final void update(Entity... entities) {
 
   @SafeVarargs
   @Override
-  public final void put(Entity... entities) {
+  public final void put(Entity... entities) {
     validateActive();
-    for (Entity entity : entities) {
+    for (Entity entity : entities) {
       Key key = entity.key();
       toAdd.remove(key);
       toUpdate.remove(key);
@@ -158,19 +165,19 @@ protected String name() {
     return name;
   }
 
-  protected Map toAdd() {
+  protected Map> toAdd() {
     return toAdd;
   }
 
-  protected List> toAddAutoId() {
+  protected List> toAddAutoId() {
     return toAddAutoId;
   }
 
-  protected Map toUpdate() {
+  protected Map> toUpdate() {
     return toUpdate;
   }
 
-  protected Map toPut() {
+  protected Map> toPut() {
     return toPut;
   }
 
@@ -194,16 +201,16 @@ protected DatastoreServiceException newInvalidRequest(String msg, Object... para
 
   protected DatastoreV1.Mutation.Builder toMutationPb() {
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-    for (Entity entity : toAddAutoId()) {
+    for (FullEntity entity : toAddAutoId()) {
       mutationPb.addInsertAutoId(entity.toPb());
     }
-    for (Entity entity : toAdd().values()) {
+    for (FullEntity entity : toAdd().values()) {
       mutationPb.addInsert(entity.toPb());
     }
-    for (Entity entity : toUpdate().values()) {
+    for (FullEntity entity : toUpdate().values()) {
       mutationPb.addUpdate(entity.toPb());
     }
-    for (Entity entity : toPut().values()) {
+    for (FullEntity entity : toPut().values()) {
       mutationPb.addUpsert(entity.toPb());
     }
     for (Key key : toDelete()) {
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index cea40428f33b..3a407249f47b 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -29,38 +29,87 @@
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Maps;
+import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 /**
- * A base class for entities to hold the properties.
+ * A base class for entities (key and properties).
+ * An entity is Google Cloud Datastore persistent data object.
+ * An entity holds one or more properties, represented by a name (as {@link String})
+ * and a value (as {@link com.google.gcloud.datastore.Value}), and may be associated with a
+ * key. For a list of possible values see {@link com.google.gcloud.datastore.Value.Type}.
+ *
+ * @see Google Cloud Datastore Entities, Properties, and Keys
  */
-abstract class BaseEntity extends Serializable {
+public abstract class BaseEntity extends Serializable {
 
   private static final long serialVersionUID = 8175618724683792766L;
 
   private final transient ImmutableSortedMap> properties;
+  private final K key;
 
-  abstract static class Builder> {
+  abstract static class Builder> {
 
-    final Map> properties;
+    private K key;
+    private final Map> properties = new HashMap<>();
 
     Builder() {
-      properties = new HashMap<>();
     }
 
-    Builder(BaseEntity entity) {
-      properties = new HashMap<>(entity.properties());
+    Builder(K key) {
+      key(key);
+    }
+
+    Builder(BaseEntity entity) {
+      this(entity.key, entity);
+    }
+
+    Builder(K key, BaseEntity entity) {
+      key(key);
+      properties(entity.properties);
+    }
+
+    protected K key() {
+      return key;
+    }
+
+    protected Map> properties() {
+      return properties;
     }
 
     @SuppressWarnings("unchecked")
-    B self() {
+    private B self() {
       return (B) this;
     }
 
+    protected B fill(DatastoreV1.Entity entityPb) {
+      Map> copiedProperties = Maps.newHashMap();
+      for (DatastoreV1.Property property : entityPb.getPropertyList()) {
+        copiedProperties.put(property.getName(), Value.fromPb(property.getValue()));
+      }
+      properties(copiedProperties);
+      if (entityPb.hasKey()) {
+        key((K) IncompleteKey.fromPb(entityPb.getKey()));
+      }
+      return self();
+    }
+
+    protected B properties(Map> properties) {
+      this.properties.putAll(properties);
+      return self();
+    }
+
+    public B key(K key) {
+      this.key = key;
+      return self();
+    }
+
     /**
      * Clears all the properties.
      */
@@ -112,7 +161,7 @@ public B set(String name, Key value) {
       return self();
     }
 
-    public B set(String name, Entity value) {
+    public B set(String name, FullEntity value) {
       properties.put(name, of(value));
       return self();
     }
@@ -140,8 +189,46 @@ public B setNull(String name) {
     public abstract BaseEntity build();
   }
 
-  BaseEntity(ImmutableSortedMap> properties) {
-    this.properties = properties;
+  BaseEntity(Builder builder) {
+    this.key = builder.key;
+    this.properties = ImmutableSortedMap.copyOf(builder.properties);
+  }
+
+  BaseEntity(BaseEntity from) {
+    this.key = from.key();
+    this.properties = from.properties;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(key, properties);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == this) {
+      return true;
+    }
+    if (!(obj instanceof BaseEntity)) {
+      return false;
+    }
+    BaseEntity other = (BaseEntity) obj;
+    return Objects.equals(key, other.key)
+        && Objects.equals(properties, other.properties);
+  }
+
+  /**
+   * Returns true if entity has a non-null key.
+   */
+  public boolean hasKey() {
+    return key != null;
+  }
+
+  /**
+   * Returns the associated key or null if it does not have one.
+   */
+  public K key() {
+    return key;
   }
 
   /**
@@ -165,50 +252,110 @@ public > V getValue(String name) {
     return property;
   }
 
+  /**
+   * Returns true if property is instanceof NullValue.
+   *
+   * @throws DatastoreServiceException if not such property.
+   */
   public boolean isNull(String name) {
     return getValue(name) instanceof NullValue;
   }
 
+
+  /**
+   * Returns the property value as a string.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a string.
+   */
   @SuppressWarnings("unchecked")
   public String getString(String name) {
     return ((Value) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as long.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a long.
+   */
   @SuppressWarnings("unchecked")
   public long getLong(String name) {
     return ((Value) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as a double.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a double.
+   */
   @SuppressWarnings("unchecked")
   public double getDouble(String name) {
     return ((Value) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as a boolean.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a boolean.
+   */
   @SuppressWarnings("unchecked")
   public boolean getBoolean(String name) {
     return ((Value) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as a DateTime.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a DateTime.
+   */
   @SuppressWarnings("unchecked")
   public DateTime getDateTime(String name) {
     return ((Value) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as a Key.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a Key.
+   */
   @SuppressWarnings("unchecked")
   public Key getKey(String name) {
     return ((Value) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as an entity.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not an entity.
+   */
   @SuppressWarnings("unchecked")
-  public  Entity getEntity(String name) {
-    return ((Value>) getValue(name)).get();
+  public  FullEntity getEntity(String name) {
+    return ((Value>) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as a list of values.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a list of values.
+   */
   @SuppressWarnings("unchecked")
   public List> getList(String name) {
     return ((Value>>) getValue(name)).get();
   }
 
+  /**
+   * Returns the property value as a blob.
+   *
+   * @throws DatastoreServiceException if not such property.
+   * @throws ClassCastException if value is not a blob.
+   */
   @SuppressWarnings("unchecked")
   public Blob getBlob(String name) {
     return ((Value) getValue(name)).get();
@@ -225,6 +372,15 @@ ImmutableSortedMap> properties() {
     return properties;
   }
 
+  @Override
+  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
+    Builder builder = emptyBuilder();
+    builder.fill(DatastoreV1.Entity.parseFrom(bytesPb));
+    return builder.build();
+  }
+
+  protected abstract Builder emptyBuilder();
+
   @Override
   protected final DatastoreV1.Entity toPb() {
     DatastoreV1.Entity.Builder entityPb = DatastoreV1.Entity.newBuilder();
@@ -234,9 +390,9 @@ protected final DatastoreV1.Entity toPb() {
       propertyPb.setValue(entry.getValue().toPb());
       entityPb.addProperty(propertyPb.build());
     }
-    populateEntityBuilder(entityPb);
+    if (key != null) {
+      entityPb.setKey(key.toPb());
+    }
     return entityPb.build();
   }
-
-  protected abstract void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb);
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
index 59b57c0ba0d3..7e1ee6d57128 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
@@ -27,14 +27,14 @@ interface DatastoreBatchWriter extends DatastoreWriter {
   /**
    * Datastore add operation.
    * This method will also allocate id for any entity with an incomplete key.
-   * As oppose to {@link #add(Entity)}, this method will defer any necessary id allocation
+   * As oppose to {@link #add(FullEntity)}, this method will defer any necessary id allocation
    * to submit time.
    *
    * @throws IllegalArgumentException if any of the given entities is missing a key
    * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity with a
    *     complete key was already added to this writer or if not active
    */
-  void addWithDeferredIdAllocation(Entity... entity);
+  void addWithDeferredIdAllocation(FullEntity... entity);
 
   /**
    * {@inheritDoc}
@@ -45,7 +45,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    *     if id allocation for an entity with an incomplete key failed.
    */
   @Override
-  List> add(Entity... entity);
+  List add(FullEntity... entity);
 
   /**
    * {@inheritDoc}
@@ -55,7 +55,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    *     deletion in this writer or if not active
    */
   @Override
-  void update(Entity... entity);
+  void update(Entity... entity);
 
   /**
    * {@inheritDoc}
@@ -72,7 +72,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    * @throws com.google.gcloud.datastore.DatastoreServiceException if not active
    */
   @Override
-  void put(Entity... entity);
+  void put(Entity... entity);
 
   /**
    * Returns {@code true} if still active (write operations were not sent to the Datastore).
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index ac7671760d68..d51147b4d053 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -38,26 +38,26 @@ static Key allocateId(DatastoreService service, IncompleteKey key) {
     return service.allocateId(new IncompleteKey[]{key}).get(0);
   }
 
-  static Entity get(DatastoreReader reader, Key key) {
+  static Entity get(DatastoreReader reader, Key key) {
     return Iterators.getNext(reader.get(new Key[]{key}), null);
   }
 
-  static Entity add(DatastoreWriter writer, Entity entity) {
-    return writer.add(new Entity[] {entity}).get(0);
+  static Entity add(DatastoreWriter writer, FullEntity entity) {
+    return writer.add(new FullEntity[] {entity}).get(0);
   }
 
   /**
    * Returns a list with a value for each given key (ordered by input).
    * A {@code null} would be returned for non-existing keys.
    */
-  static List> fetch(DatastoreReader reader, Key... keys) {
-    Iterator> entities = reader.get(keys);
-    Map> map = Maps.newHashMapWithExpectedSize(keys.length);
+  static List fetch(DatastoreReader reader, Key... keys) {
+    Iterator entities = reader.get(keys);
+    Map map = Maps.newHashMapWithExpectedSize(keys.length);
     while (entities.hasNext()) {
-      Entity entity = entities.next();
+      Entity entity = entities.next();
       map.put(entity.key(), entity);
     }
-    List> list = new ArrayList<>(keys.length);
+    List list = new ArrayList<>(keys.length);
     for (Key key : keys) {
       // this will include nulls for non-existing keys
       list.add(map.get(key));
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index 2bbd2764dc35..b1dc0a70d392 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -29,7 +29,7 @@ public interface DatastoreReader {
    *
    * @throws DatastoreServiceException upon failure.
    */
-  Entity get(Key key);
+  Entity get(Key key);
 
   /**
    * Returns an {@link Entity} for each given {@link Key} that exists in the Datastore.
@@ -41,14 +41,14 @@ public interface DatastoreReader {
    * @throws DatastoreServiceException upon failure.
    * @see #get(Key)
    */
-  Iterator> get(Key... key);
+  Iterator get(Key... key);
 
   /**
    * Returns a list with a value for each given key (ordered by input).
    * A {@code null} would be returned for non-existing keys.
    * When possible prefer using {@link #get(Key...)} which does not eagerly loads the results.
    */
-  List> fetch(Key... keys);
+  List fetch(Key... keys);
 
   /**
    * Submit a {@link Query} and returns its result.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index 464d2308bbe3..c6401c6a0582 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -86,14 +86,14 @@ interface TransactionCallable {
    * @throws DatastoreServiceException upon failure
    */
   @Override
-  void update(Entity... entity);
+  void update(Entity... entity);
 
   /**
    * {@inheritDoc}
    * @throws DatastoreServiceException upon failure
    */
   @Override
-  void put(Entity... entity);
+  void put(Entity... entity);
 
   /**
    * {@inheritDoc}
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 6fadadf2f831..98dcdf0f32da 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -162,22 +162,22 @@ private IncompleteKey trimNameOrId(IncompleteKey key) {
   }
 
   @Override
-  public Entity add(Entity entity) {
+  public Entity add(FullEntity entity) {
     return DatastoreHelper.add(this, entity);
   }
 
   @SuppressWarnings("unchecked")
   @Override
-  public List> add(Entity... entities) {
+  public List add(FullEntity... entities) {
     if (entities.length == 0) {
       return Collections.emptyList();
     }
     DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
     Map completeEntities = new LinkedHashMap<>();
-    for (Entity entity : entities) {
-      Entity completeEntity = null;
+    for (FullEntity entity : entities) {
+      Entity completeEntity = null;
       if (entity.key() instanceof Key) {
-        completeEntity = (Entity) entity;
+        completeEntity = Entity.convert((FullEntity) entity);
       }
       if (completeEntity != null) {
         if (completeEntities.put(completeEntity.key(), completeEntity) != null) {
@@ -193,10 +193,9 @@ public List> add(Entity... entities) {
     DatastoreV1.CommitResponse commitResponse = commitMutation(mutationPb);
     Iterator allocatedKeys =
         commitResponse.getMutationResult().getInsertAutoIdKeyList().iterator();
-    ImmutableList.Builder> responseBuilder = ImmutableList.builder();
-    for (Entity entity : entities) {
-      IncompleteKey key = entity.key();
-      Entity completeEntity = completeEntities.get(key);
+    ImmutableList.Builder responseBuilder = ImmutableList.builder();
+    for (FullEntity entity : entities) {
+      Entity completeEntity = completeEntities.get(entity.key());
       if (completeEntity != null) {
         responseBuilder.add(completeEntity);
       } else {
@@ -207,21 +206,21 @@ public List> add(Entity... entities) {
   }
 
   @Override
-  public Entity get(Key key) {
+  public Entity get(Key key) {
     return DatastoreHelper.get(this, key);
   }
 
   @Override
-  public Iterator> get(Key... keys) {
+  public Iterator get(Key... keys) {
     return get(null, keys);
   }
 
   @Override
-  public List> fetch(Key... keys) {
+  public List fetch(Key... keys) {
     return DatastoreHelper.fetch(this, keys);
   }
 
-  Iterator> get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) {
+  Iterator get(DatastoreV1.ReadOptions readOptionsPb, final Key... keys) {
     if (keys.length == 0) {
       return Collections.emptyIterator();
     }
@@ -235,7 +234,7 @@ Iterator> get(DatastoreV1.ReadOptions readOptionsPb, final Key... ke
     return new ResultsIterator(requestPb);
   }
 
-  final class ResultsIterator extends AbstractIterator> {
+  final class ResultsIterator extends AbstractIterator {
 
     private final DatastoreV1.LookupRequest.Builder requestPb;
     Iterator iter;
@@ -256,7 +255,7 @@ private void loadResults() {
 
     @SuppressWarnings("unchecked")
     @Override
-    protected Entity computeNext() {
+    protected Entity computeNext() {
       if (iter.hasNext()) {
         return Entity.fromPb(iter.next().getEntity());
       }
@@ -284,11 +283,11 @@ DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) {
 
   @SafeVarargs
   @Override
-  public final void update(Entity... entities) {
+  public final void update(Entity... entities) {
     if (entities.length > 0) {
       DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-      Map> dedupEntities = new LinkedHashMap<>();
-      for (Entity entity : entities) {
+      Map dedupEntities = new LinkedHashMap<>();
+      for (Entity entity : entities) {
         dedupEntities.put(entity.key(), entity);
       }
       for (Entity entity : dedupEntities.values()) {
@@ -300,11 +299,11 @@ public final void update(Entity... entities) {
 
   @SafeVarargs
   @Override
-  public final void put(Entity... entities) {
+  public final void put(Entity... entities) {
     if (entities.length > 0) {
       DatastoreV1.Mutation.Builder mutationPb = DatastoreV1.Mutation.newBuilder();
-      Map> dedupEntities = new LinkedHashMap<>();
-      for (Entity entity : entities) {
+      Map dedupEntities = new LinkedHashMap<>();
+      for (Entity entity : entities) {
         dedupEntities.put(entity.key(), entity);
       }
       for (Entity e : dedupEntities.values()) {
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
index 255ab8fa8a04..806394552419 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
@@ -33,31 +33,31 @@ public interface DatastoreWriter {
    * @throws DatastoreServiceException upon failure
    * @throws IllegalArgumentException if the given entity is missing a key
    */
-  Entity add(Entity entity);
+  Entity add(FullEntity entity);
 
   /**
    * Datastore add operation.
    * This method will automatically allocate id for any entity with an incomplete key.
    *
-   * @return a list of {@code Entity} ordered by input with the same properties and a key that
+   * @return a list of {@code Entity} ordered by input with the same properties and a key that
    *     is either newly allocated or the same one if was already complete
    * @throws DatastoreServiceException upon failure
    * @throws IllegalArgumentException if any of the given entities is missing a key
-   * @see #add(Entity)
+   * @see #add(FullEntity)
    */
-  List> add(Entity... entity);
+  List add(FullEntity... entity);
 
   /**
    * A Datastore update operation.
    * The operation will fail if an entity with the same key does not already exist.
    */
-  void update(Entity... entity);
+  void update(Entity... entity);
 
   /**
    * A Datastore put (a.k.a upsert) operation.
    * The operation will add or modify the entities.
    */
-  void put(Entity... entity);
+  void put(Entity... entity);
 
   /**
    * A datastore delete operation.
diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/src/main/java/com/google/gcloud/datastore/Entity.java
index 62a3141ce368..7842fba61f0c 100644
--- a/src/main/java/com/google/gcloud/datastore/Entity.java
+++ b/src/main/java/com/google/gcloud/datastore/Entity.java
@@ -19,132 +19,81 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-import java.util.Objects;
+import com.google.common.base.Preconditions;
 
 /**
- * An entity is the Google Cloud Datastore persistent data object.
- * An entity holds one or more properties, represented by a name (as {@link String})
- * and a value (as {@link Value}), and is associated with a {@link Key}.
- * For a list of possible values see {@link Value.Type}.
- * This class is immutable.
- *
- * @see Google Cloud Datastore Entities, Properties, and Keys
+ * An entity is the Google Cloud Datastore persistent data object for a specific key.
+ * An entity will always have a complete {@link Key}.
  */
-public final class Entity extends BaseEntity {
+public final class Entity extends FullEntity {
 
   private static final long serialVersionUID = 432961565733066915L;
 
-  private final K key;
-
-  public static final class Builder
-      extends BaseEntity.Builder> {
-
-    private K key;
+  public static final class Builder extends BaseEntity.Builder {
 
     private Builder() {
     }
 
-    private Builder(K key) {
-      this.key = checkNotNull(key);
+    private Builder(Key key) {
+      super(checkNotNull(key));
     }
 
-    private Builder(Entity entity) {
+    private Builder(Entity entity) {
       super(entity);
-      key = entity.key();
     }
 
-    private Builder(K key, BaseEntity entity) {
-      super(entity);
-      this.key = checkNotNull(key);
+    private Builder(Key key, FullEntity entity) {
+      properties(entity.properties());
+      key(key);
     }
 
-    public Builder key(K key) {
-      this.key = checkNotNull(key);
+    @Override
+    public Builder key(Key key) {
+      super.key(checkNotNull(key));
       return this;
     }
 
     @Override
-    public Entity build() {
-      return new Entity<>(key, ImmutableSortedMap.copyOf(properties));
+    public Entity build() {
+      Preconditions.checkState(key() != null);
+      return new Entity(this);
     }
   }
 
-  Entity(K key, ImmutableSortedMap> properties) {
-    super(properties);
-    this.key = key;
+  Entity(Builder builder) {
+    super(builder);
   }
 
-  @Override
-  public int hashCode() {
-    return Objects.hash(key, properties());
+  Entity(FullEntity from) {
+    super(from);
+    Preconditions.checkArgument(from.key() != null);
   }
 
   @Override
-  public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
-    }
-    if (!(obj instanceof Entity)) {
-      return false;
-    }
-    Entity other = (Entity) obj;
-    return Objects.equals(key, other.key)
-        && Objects.equals(properties(), other.properties());
+  protected BaseEntity.Builder emptyBuilder() {
+    return new Builder();
   }
 
-  @Override
-  protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {
-    if (key != null) {
-      entityPb.setKey(key.toPb());
+  static Entity convert(FullEntity from) {
+    if (from instanceof Entity) {
+      return (Entity) from;
     }
+    return new Entity(from);
   }
 
-  public boolean hasKey() {
-    return key != null;
+  public static Builder builder(Key key) {
+    return new Builder(key);
   }
 
-  /**
-   * Returns the entity's key.
-   * Entity will always have a key where as Entity may or may not have a key.
-   */
-  public K key() {
-    return key;
+  public static Builder builder(Entity copyFrom) {
+    return new Builder(copyFrom);
   }
 
-  @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    return fromPb(DatastoreV1.Entity.parseFrom(bytesPb));
+  public static Builder builder(Key key, FullEntity copyFrom) {
+    return new Builder(key, copyFrom);
   }
 
   static Entity fromPb(DatastoreV1.Entity entityPb) {
-    ImmutableSortedMap.Builder> properties =
-        ImmutableSortedMap.naturalOrder();
-    for (DatastoreV1.Property property : entityPb.getPropertyList()) {
-      properties.put(property.getName(), Value.fromPb(property.getValue()));
-    }
-    IncompleteKey key = null;
-    if (entityPb.hasKey()) {
-      key = IncompleteKey.fromPb(entityPb.getKey());
-    }
-    return new Entity<>(key, properties.build());
-  }
-
-  public static Builder builder() {
-    return new Builder<>();
-  }
-
-  public static  Builder builder(K key) {
-    return new Builder<>(key);
-  }
-
-  public static  Builder builder(Entity copyFrom) {
-    return new Builder<>(copyFrom);
-  }
-
-  public static  Builder builder(K key, Entity copyFrom) {
-    return new Builder<>(key, copyFrom);
+    return new Builder().fill(entityPb).build();
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java
index 68478c8d97cb..44828fc35839 100644
--- a/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -21,12 +21,12 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Preconditions;
 
-public class EntityValue extends Value> {
+public class EntityValue extends Value {
 
   private static final long serialVersionUID = -5461475706792576395L;
 
-  static final BaseMarshaller, EntityValue, Builder> MARSHALLER =
-      new BaseMarshaller, EntityValue, Builder>() {
+  static final BaseMarshaller MARSHALLER =
+      new BaseMarshaller() {
 
         private static final long serialVersionUID = 2355075086076070931L;
 
@@ -36,13 +36,13 @@ public int getProtoFieldId() {
         }
 
         @Override
-        public Builder newBuilder(Entity value) {
+        public Builder newBuilder(FullEntity value) {
           return builder(value);
         }
 
         @Override
-        protected Entity getValue(DatastoreV1.Value from) {
-          return Entity.fromPb(from.getEntityValue());
+        protected FullEntity getValue(DatastoreV1.Value from) {
+          return FullEntity.fromPb(from.getEntityValue());
         }
 
         @Override
@@ -51,7 +51,7 @@ protected void setValue(EntityValue from, DatastoreV1.Value.Builder to) {
         }
       };
 
-  public static final class Builder extends Value.BaseBuilder, EntityValue, Builder> {
+  public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
       super(Type.ENTITY);
@@ -70,7 +70,7 @@ public EntityValue build() {
     }
   }
 
-  public EntityValue(Entity entity) {
+  public EntityValue(FullEntity entity) {
     this(builder(entity));
   }
 
@@ -83,11 +83,11 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
-  public static EntityValue of(Entity entity) {
+  public static EntityValue of(FullEntity entity) {
     return new EntityValue(entity);
   }
 
-  public static Builder builder(Entity entity) {
+  public static Builder builder(FullEntity entity) {
     return new Builder().set(entity).indexed(false);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/FullEntity.java b/src/main/java/com/google/gcloud/datastore/FullEntity.java
new file mode 100644
index 000000000000..ac20c50136b4
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/FullEntity.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import com.google.api.services.datastore.DatastoreV1;
+
+/**
+ * A full entity is a {@link BaseEntity} that with a complete set of properties.
+ */
+public class FullEntity extends BaseEntity {
+
+  private static final long serialVersionUID = 432961565733066915L;
+
+  public static class Builder extends BaseEntity.Builder> {
+
+    Builder() {
+    }
+
+    Builder(K key) {
+      super(key);
+    }
+
+    Builder(FullEntity entity) {
+      super(entity);
+    }
+
+    @Override
+    public FullEntity build() {
+      return new FullEntity<>(this);
+    }
+  }
+
+  FullEntity(BaseEntity.Builder builder) {
+    super(builder);
+  }
+
+  FullEntity(FullEntity from) {
+    super(from);
+  }
+
+  @Override
+  protected BaseEntity.Builder emptyBuilder() {
+    return new Builder();
+  }
+
+  public static Builder builder() {
+    return new Builder<>();
+  }
+
+  public static  Builder builder(K key) {
+    return new Builder<>(key);
+  }
+
+  public static  Builder builder(FullEntity copyFrom) {
+    return new Builder<>(copyFrom);
+  }
+
+
+  static FullEntity fromPb(DatastoreV1.Entity entityPb) {
+    return new Builder<>().fill(entityPb).build();
+  }
+}
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index cd70d35ac9f6..1ebb313cdd26 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -227,7 +227,7 @@ public Builder setBinding(String name, Key... value) {
       return this;
     }
 
-    public Builder setBinding(String name, Entity... value) {
+    public Builder setBinding(String name, FullEntity... value) {
       namedBindings.put(name, toBinding(name, EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
@@ -272,7 +272,7 @@ public Builder addBinding(Key... value) {
       return this;
     }
 
-    public Builder addBinding(Entity... value) {
+    public Builder addBinding(FullEntity... value) {
       positionalBindings.add(toBinding(EntityValue.MARSHALLER, Arrays.asList(value)));
       return this;
     }
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
index 1aa95b37110c..acd9783fe836 100644
--- a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -17,11 +17,7 @@
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.common.collect.ImmutableSortedMap;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-import java.util.Objects;
 
 /**
  * A projection entity is a result of a Google Cloud Datastore projection query.
@@ -31,67 +27,27 @@
  * @see Google Cloud Datastore projection queries
  * @see Google Cloud Datastore Entities, Properties, and Keys
  */
-public final class ProjectionEntity extends BaseEntity {
+public final class ProjectionEntity extends BaseEntity {
 
   private static final long serialVersionUID = 432961565733066915L;
 
-  private final Key key;
-
-  static final class Builder extends BaseEntity.Builder {
-
-    private Key key;
+  static final class Builder extends BaseEntity.Builder {
 
     Builder() {
     }
 
     private Builder(ProjectionEntity entity) {
       super(entity);
-      key = entity.key();
-    }
-
-    public Builder key(Key key) {
-      this.key = key;
-      return this;
     }
 
     @Override
     public ProjectionEntity build() {
-      return new ProjectionEntity(key, ImmutableSortedMap.copyOf(properties));
-    }
-  }
-
-  ProjectionEntity(Key key, ImmutableSortedMap> properties) {
-    super(properties);
-    this.key = key;
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(key, properties());
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (obj == this) {
-      return true;
+      return new ProjectionEntity(this);
     }
-    if (!(obj instanceof ProjectionEntity)) {
-      return false;
-    }
-    ProjectionEntity other = (ProjectionEntity) obj;
-    return Objects.equals(key, other.key)
-        && Objects.equals(properties(), other.properties());
-  }
-
-  public boolean hasKey() {
-    return key() != null;
   }
 
-  /**
-   * Returns the associated {@link Key} or null if it does not have one.
-   */
-  public Key key() {
-    return key;
+  ProjectionEntity(Builder builder) {
+    super(builder);
   }
 
   @SuppressWarnings({"unchecked", "deprecation"})
@@ -114,37 +70,16 @@ public Blob getBlob(String name) {
     return ((Value) value).get();
   }
 
-  @Override
-  protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    return fromPb(DatastoreV1.Entity.parseFrom(bytesPb));
-  }
-
   static ProjectionEntity fromPb(DatastoreV1.Entity entityPb) {
-    ImmutableSortedMap.Builder> properties =
-        ImmutableSortedMap.naturalOrder();
-    for (DatastoreV1.Property property : entityPb.getPropertyList()) {
-      properties.put(property.getName(), Value.fromPb(property.getValue()));
-    }
-    Key key = null;
-    if (entityPb.hasKey()) {
-      key = Key.fromPb(entityPb.getKey());
-    }
-    return new ProjectionEntity(key, properties.build());
-  }
-
-
-  public static Builder builder(ProjectionEntity copyFrom) {
-    return new Builder(copyFrom);
+    return new Builder().fill(entityPb).build();
   }
 
-  static Builder builder() {
+  @Override
+  protected Builder emptyBuilder() {
     return new Builder();
   }
 
-  @Override
-  protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {
-    if (key != null) {
-      entityPb.setKey(key.toPb());
-    }
+  public static Builder builder(ProjectionEntity copyFrom) {
+    return new Builder(copyFrom);
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index c5b8705ee5c7..83b5396cd4c5 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -41,17 +41,13 @@
 public abstract class Query extends Serializable {
 
   private static final long serialVersionUID = -2748141759901313101L;
-  private static final Object OBJECT_INSTANCE = new Object();
-  private static final Key KEY_INSTANCE = Key.builder("dummy", "dummy", "dummy").build();
-  private static final Entity ENTITY_INSTANCE = Entity.builder(KEY_INSTANCE).build();
-  private static final ProjectionEntity PROJECTION_ENTITY = ProjectionEntity.builder().build();
 
   private final Type type;
   private final String namespace;
 
   /**
    * This class represents the expected type of the result.
-   *   FULL: A complete {@link Entity}.
+   *   FULL: A full entity represented by {@link Entity}.
    *   PROJECTION: A projection entity, represented by {@link ProjectionEntity}.
    *   KEY_ONLY: An entity's {@link Key}.
    */
@@ -61,7 +57,7 @@ public abstract static class Type implements java.io.Serializable {
     private static final Map>
         PB_TO_INSTANCE = Maps.newEnumMap(DatastoreV1.EntityResult.ResultType.class);
 
-    static final Type UNKNOWN = new Type(null, OBJECT_INSTANCE) {
+    static final Type UNKNOWN = new Type(null, Object.class) {
 
       private static final long serialVersionUID = 1602329532153860907L;
 
@@ -76,20 +72,18 @@ public abstract static class Type implements java.io.Serializable {
       }
     };
 
-    public static final Type> FULL =
-        new Type>(DatastoreV1.EntityResult.ResultType.FULL, ENTITY_INSTANCE) {
+    public static final Type FULL =
+        new Type(DatastoreV1.EntityResult.ResultType.FULL, Entity.class) {
 
       private static final long serialVersionUID = 7712959777507168274L;
 
-      @SuppressWarnings("unchecked")
-      @Override
-      protected Entity convert(DatastoreV1.Entity entityPb) {
+      @Override protected Entity convert(DatastoreV1.Entity entityPb) {
         return Entity.fromPb(entityPb);
       }
     };
 
     public static final Type KEY_ONLY =
-        new Type(DatastoreV1.EntityResult.ResultType.KEY_ONLY, KEY_INSTANCE) {
+        new Type(DatastoreV1.EntityResult.ResultType.KEY_ONLY, Key.class) {
 
       private static final long serialVersionUID = -8514289244104446252L;
 
@@ -99,7 +93,7 @@ protected Entity convert(DatastoreV1.Entity entityPb) {
     };
 
     public static final Type PROJECTION = new Type(
-        DatastoreV1.EntityResult.ResultType.PROJECTION, PROJECTION_ENTITY) {
+        DatastoreV1.EntityResult.ResultType.PROJECTION, ProjectionEntity.class) {
 
           private static final long serialVersionUID = -7591409419690650246L;
 
@@ -112,9 +106,9 @@ protected Entity convert(DatastoreV1.Entity entityPb) {
     private final DatastoreV1.EntityResult.ResultType resultType;
 
     @SuppressWarnings("unchecked")
-    private Type(DatastoreV1.EntityResult.ResultType resultType, V resultObject) {
+    private Type(DatastoreV1.EntityResult.ResultType resultType, Class resultClass) {
       this.resultType = resultType;
-      this.resultClass = (Class) checkNotNull(resultObject).getClass();
+      this.resultClass = resultClass;
       if (resultType != null) {
         PB_TO_INSTANCE.put(resultType, this);
       }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 0eceeeffd6f8..73cc7edde09b 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -48,9 +48,9 @@
  * 
A simple query that returns all entities for a specific kind
  * 
{@code
  *   FullQuery query = Query.fullQueryBuilder().kind(kind).build();
- *   QueryResults> results = datastore.run(query);
+ *   QueryResults results = datastore.run(query);
  *   while (results.hasNext()) {
- *     Entity entity = results.next();
+ *     Entity entity = results.next();
  *     ...
  *   }
  * } 

@@ -753,7 +753,7 @@ static final class Builder extends BaseBuilder> {
     }
   }
 
-  public static final class FullQueryBuilder extends BaseBuilder, FullQueryBuilder> {
+  public static final class FullQueryBuilder extends BaseBuilder {
 
     FullQueryBuilder() {
       super(Type.FULL);
@@ -829,9 +829,9 @@ public ProjectionQueryBuilder addGroupBy(String property, String... others) {
     }
   }
 
-  public static final class FullQuery extends StructuredQuery> {
+  public static final class FullQuery extends StructuredQuery {
 
-    FullQuery(BaseBuilder, ?> builder) {
+    FullQuery(BaseBuilder builder) {
       super(builder);
     }
   }
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/src/main/java/com/google/gcloud/datastore/Transaction.java
index bd2866da142b..6ac1da78ab75 100644
--- a/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -65,7 +65,7 @@ interface Response {
    * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
-  Entity get(Key key);
+  Entity get(Key key);
 
   /**
    * {@inheritDoc}
@@ -76,7 +76,7 @@ interface Response {
    * @throws DatastoreServiceException upon failure or if no longer active
    */
   @Override
-  Iterator> get(Key... key);
+  Iterator get(Key... key);
 
   /**
    * {@inheritDoc}
@@ -86,7 +86,8 @@ interface Response {
    *
    * @throws DatastoreServiceException upon failure or if no longer active
    */
-  List> fetch(Key... keys);
+  @Override
+  List fetch(Key... keys);
 
   /**
    * {@inheritDoc}
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index df2fab4134e8..6f1090945ee1 100644
--- a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -70,12 +70,12 @@ public List generatedKeys() {
   }
 
   @Override
-  public Entity get(Key key) {
+  public Entity get(Key key) {
     return DatastoreHelper.get(this, key);
   }
 
   @Override
-  public Iterator> get(Key... keys) {
+  public Iterator get(Key... keys) {
     validateActive();
     DatastoreV1.ReadOptions.Builder readOptionsPb = DatastoreV1.ReadOptions.newBuilder();
     readOptionsPb.setTransaction(transaction);
@@ -83,7 +83,7 @@ public Iterator> get(Key... keys) {
   }
 
   @Override
-  public List> fetch(Key... keys) {
+  public List fetch(Key... keys) {
     validateActive();
     return DatastoreHelper.fetch(this, keys);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index e9ccd388351c..7661deebb82c 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -23,7 +23,7 @@
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
  * KeyFactory keyFactory = new KeyFactory(datastore).kind(kind);
  * Key key = keyFactory.newKey(keyName);
- * Entity<Key> entity = datastore.get(key);
+ * Entity entity = datastore.get(key);
  * if (entity == null) {
  *   entity = Entity.builder(key)
  *       .set("name", "John Do")
diff --git a/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
index 2169f7b9ead8..71576036fe3b 100644
--- a/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.List;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder(KEY1, 1).build();
  private static final Key KEY3 = Key.builder(KEY1, 2).build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder(KEY1).build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final Entity INCOMPLETE_ENTITY_1 =
      Entity.builder(INCOMPLETE_KEY).build();
  private static final Entity INCOMPLETE_ENTITY_2 =
      Entity.builder(INCOMPLETE_KEY).set("name", "dan").build();

  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    private final DatastoreService datastore;

    protected DatastoreBatchWriter() {
      super("test");
      datastore = EasyMock.createMock(DatastoreService.class);
      IncompleteKey[] expected = {INCOMPLETE_KEY, INCOMPLETE_KEY};
      List result = ImmutableList.of(KEY2, KEY3);
      expect(datastore.allocateId(expected)).andReturn(result).times(0, 1);
      replay(datastore);
    }

    @Override
    protected DatastoreService datastore() {
      return datastore;
    }

    void finish() {
      verify(datastore);
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @After
  public void tearDown() {
    batchWriter.finish();
  }

  @Test
  public void testAdd() throws Exception {
    Entity entity2 =
        Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(entity2.toPb())
        .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb())
        .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb())
        .build();
    List> entities = batchWriter
        .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2);
    assertEquals(pb, batchWriter.toMutationPb().build());
    assertEquals(ENTITY1, entities.get(0));
    assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1));
    assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2));
    assertEquals(entity2, entities.get(3));
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddWithDeferredAllocation() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb())
        .build();
    batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWithDeferredAllocationWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.List;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder(KEY1, 1).build();
  private static final Key KEY3 = Key.builder(KEY1, 2).build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder(KEY1).build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final FullEntity INCOMPLETE_ENTITY_1 =
      Entity.builder(INCOMPLETE_KEY).build();
  private static final FullEntity INCOMPLETE_ENTITY_2 =
      Entity.builder(INCOMPLETE_KEY).set("name", "dan").build();

  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    private final DatastoreService datastore;

    protected DatastoreBatchWriter() {
      super("test");
      datastore = EasyMock.createMock(DatastoreService.class);
      IncompleteKey[] expected = {INCOMPLETE_KEY, INCOMPLETE_KEY};
      List result = ImmutableList.of(KEY2, KEY3);
      expect(datastore.allocateId(expected)).andReturn(result).times(0, 1);
      replay(datastore);
    }

    @Override
    protected DatastoreService datastore() {
      return datastore;
    }

    void finish() {
      verify(datastore);
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @After
  public void tearDown() {
    batchWriter.finish();
  }

  @Test
  public void testAdd() throws Exception {
    Entity entity2 =
        Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(entity2.toPb())
        .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb())
        .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb())
        .build();
    List entities = batchWriter
        .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2);
    assertEquals(pb, batchWriter.toMutationPb().build());
    assertEquals(ENTITY1, entities.get(0));
    assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1));
    assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2));
    assertEquals(entity2, entities.get(3));
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddWithDeferredAllocation() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb())
        .build();
    batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWithDeferredAllocationWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
index dba278c1857d..8a2ffb26d68e 100644
--- a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.protobuf.InvalidProtocolBufferException;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final Entity PARTIAL_ENTITY = Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override
    public BaseEntity build() {

      return new BaseEntity(ImmutableSortedMap.copyOf(properties)) {
        @Override
        protected void populateEntityBuilder(DatastoreV1.Entity.Builder entityPb) {

        }

        @Override
        protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
          return null;
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final FullEntity PARTIAL_ENTITY =
      Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override public BaseEntity build() {

      return new BaseEntity(this) {

        @Override
        protected Builder emptyBuilder() {
          return new BaseEntityTest.Builder();
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 14c9a8733499..57089b0a4f2d 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -48,12 +48,12 @@ public void testGet() throws Exception {
     DatastoreService datastoreService = createStrictMock(DatastoreService.class);
     IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
     Key key1 = Key.builder(pKey1, 1).build();
-    Entity entity1 = Entity.builder(key1).build();
+    Entity entity1 = Entity.builder(key1).build();
     Key key2 = Key.builder(pKey1, 2).build();
     expect(datastoreService.get(new Key[]{key1}))
         .andReturn(Collections.singletonList(entity1).iterator());
     expect(datastoreService.get(new Key[]{key2}))
-        .andReturn(Collections.>emptyIterator());
+        .andReturn(Collections.emptyIterator());
     replay(datastoreService);
     assertEquals(entity1, DatastoreHelper.get(datastoreService, key1));
     assertNull(DatastoreHelper.get(datastoreService, key2));
@@ -65,7 +65,7 @@ public void testAdd() throws Exception {
     DatastoreService datastoreService = createStrictMock(DatastoreService.class);
     IncompleteKey pKey = IncompleteKey.builder("ds", "k").build();
     Key key = Key.builder(pKey, 1).build();
-    Entity entity = Entity.builder(key).build();
+    Entity entity = Entity.builder(key).build();
     expect(datastoreService.add(new Entity[]{entity}))
         .andReturn(Collections.singletonList(entity));
     replay(datastoreService);
@@ -79,11 +79,11 @@ public void testFetch() throws Exception {
     IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
     Key key1 = Key.builder(pKey1, 1).build();
     Key key2 = Key.builder(pKey1, "a").build();
-    Entity entity1 = Entity.builder(key1).build();
-    Entity entity2 = Entity.builder(key2).build();
+    Entity entity1 = Entity.builder(key1).build();
+    Entity entity2 = Entity.builder(key2).build();
     expect(datastoreService.get(key1, key2)).andReturn(Iterators.forArray(entity1, entity2)).once();
     replay(datastoreService);
-    List> values = DatastoreHelper.fetch(datastoreService, key1, key2);
+    List values = DatastoreHelper.fetch(datastoreService, key1, key2);
     assertEquals(2, values.size());
     assertEquals(entity1, values.get(0));
     assertEquals(entity2, values.get(1));
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 0b8a1ea6d005..25c668f01db8 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -68,22 +68,25 @@ public class DatastoreServiceTest {
       .build();
   private static final ListValue LIST_VALUE2 = ListValue.of(Collections.singletonList(KEY_VALUE));
   private static final DateTimeValue DATE_TIME_VALUE = new DateTimeValue(DateTime.now());
-  private static final Entity PARTIAL_ENTITY1 = Entity.builder(INCOMPLETE_KEY2)
-      .set("str", STR_VALUE).set("bool", BOOL_VALUE).set("list", LIST_VALUE1).build();
-  private static final Entity PARTIAL_ENTITY2 = Entity.builder(PARTIAL_ENTITY1)
-      .remove("str").set("bool", true).set("list", LIST_VALUE1.get()).build();
-  private static final Entity PARTIAL_ENTITY3 = Entity.builder(PARTIAL_ENTITY1)
-      .key(IncompleteKey.builder(DATASET, KIND3).build()).build();
-  private static final Entity ENTITY1 = Entity.builder(KEY1)
+  private static final FullEntity PARTIAL_ENTITY1 =
+      FullEntity.builder(INCOMPLETE_KEY2).set("str", STR_VALUE).set("bool", BOOL_VALUE)
+          .set("list", LIST_VALUE1).build();
+  private static final FullEntity PARTIAL_ENTITY2 =
+      FullEntity.builder(PARTIAL_ENTITY1).remove("str").set("bool", true).
+          set("list", LIST_VALUE1.get()).build();
+  private static final FullEntity PARTIAL_ENTITY3 =
+      FullEntity.builder(PARTIAL_ENTITY1).key(IncompleteKey.builder(DATASET, KIND3).build())
+          .build();
+  private static final Entity ENTITY1 = Entity.builder(KEY1)
       .set("str", STR_VALUE)
       .set("date", DATE_TIME_VALUE)
       .set("bool", BOOL_VALUE)
       .set("partial1", EntityValue.of(PARTIAL_ENTITY1))
       .set("list", LIST_VALUE2)
       .build();
-  private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
+  private static final Entity ENTITY2 = Entity.builder(ENTITY1).key(KEY2).remove("str")
       .set("name", "Dan").setNull("null").set("age", 20).build();
-  private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
+  private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
       .set("null", NULL_VALUE).set("partial1", PARTIAL_ENTITY2).set("partial2", ENTITY2).build();
 
   private DatastoreServiceOptions options;
@@ -135,7 +138,7 @@ public void testNewTransactionCommit() {
     transaction.delete(KEY1);
     transaction.commit();
 
-    List> list = datastore.fetch(KEY1, KEY2, KEY3);
+    List list = datastore.fetch(KEY1, KEY2, KEY3);
     assertNull(list.get(0));
     assertEquals(entity2, list.get(1));
     assertEquals(ENTITY3, list.get(2));
@@ -181,12 +184,12 @@ public void testTransactionWithRead() {
 
   @Test
   public void testTransactionWithQuery() {
-    Query> query = Query.fullQueryBuilder()
+    Query query = Query.fullQueryBuilder()
         .kind(KIND2)
         .filter(PropertyFilter.hasAncestor(KEY2))
         .build();
     Transaction transaction = datastore.newTransaction();
-    QueryResults> results = transaction.run(query);
+    QueryResults results = transaction.run(query);
     assertEquals(ENTITY2, results.next());
     assertFalse(results.hasNext());
     transaction.add(ENTITY3);
@@ -227,7 +230,7 @@ public void testNewTransactionRollback() {
 
     verifyNotUsable(transaction);
 
-    List> list = datastore.fetch(KEY1, KEY2, KEY3);
+    List list = datastore.fetch(KEY1, KEY2, KEY3);
     assertEquals(ENTITY1, list.get(0));
     assertEquals(ENTITY2, list.get(1));
     assertNull(list.get(2));
@@ -267,13 +270,13 @@ private void verifyNotUsable(DatastoreWriter writer) {
   @Test
   public void testNewBatch() {
     Batch batch = datastore.newBatch();
-    Entity entity1 = Entity.builder(ENTITY1).clear().build();
-    Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
-    Entity entity4 = Entity.builder(KEY4).set("value", StringValue.of("value")).build();
-    Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
+    Entity entity1 = Entity.builder(ENTITY1).clear().build();
+    Entity entity2 = Entity.builder(ENTITY2).clear().setNull("bla").build();
+    Entity entity4 = Entity.builder(KEY4).set("value", StringValue.of("value")).build();
+    Entity entity5 = Entity.builder(KEY5).set("value", "value").build();
 
-    List> entities = batch.add(entity4, PARTIAL_ENTITY2, entity5);
-    Entity entity6 = entities.get(1);
+    List entities = batch.add(entity4, PARTIAL_ENTITY2, entity5);
+    Entity entity6 = entities.get(1);
     assertSame(entity4, entities.get(0));
     assertEquals(PARTIAL_ENTITY2.properties(), entity6.properties());
     assertEquals(PARTIAL_ENTITY2.key().dataset(), entity6.key().dataset());
@@ -324,8 +327,8 @@ public void testNewBatch() {
 
   @Test
   public void testRunGqlQueryNoCasting() {
-    Query> query1 = Query.gqlQueryBuilder(Type.FULL, "select * from " + KIND1).build();
-    QueryResults> results1 = datastore.run(query1);
+    Query query1 = Query.gqlQueryBuilder(Type.FULL, "select * from " + KIND1).build();
+    QueryResults results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
@@ -377,9 +380,9 @@ public void testRunGqlQueryNoCasting() {
   @Test
   public void testRunGqlQueryWithCasting() {
     @SuppressWarnings("unchecked")
-    Query> query1 =
-        (Query>) Query.gqlQueryBuilder("select * from " + KIND1).build();
-    QueryResults> results1 = datastore.run(query1);
+    Query query1 =
+        (Query) Query.gqlQueryBuilder("select * from " + KIND1).build();
+    QueryResults results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
@@ -398,7 +401,7 @@ public void testRunGqlQueryWithCasting() {
   public void testRunStructuredQuery() {
     FullQuery query =
         Query.fullQueryBuilder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
-    QueryResults> results1 = datastore.run(query);
+    QueryResults results1 = datastore.run(query);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
@@ -495,7 +498,7 @@ public void testGet() {
     assertEquals(LIST_VALUE2, value3);
     DateTimeValue value4 = entity.getValue("date");
     assertEquals(DATE_TIME_VALUE, value4);
-    Entity value5 = entity.getEntity("partial1");
+    FullEntity value5 = entity.getEntity("partial1");
     assertEquals(PARTIAL_ENTITY1, value5);
     assertEquals(5, entity.names().size());
     assertFalse(entity.contains("bla"));
@@ -504,7 +507,7 @@ public void testGet() {
   @Test
   public void testGetArray() {
     datastore.put(ENTITY3);
-    Iterator> result =
+    Iterator result =
         datastore.fetch(KEY1, Key.builder(KEY1).name("bla").build(), KEY2, KEY3).iterator();
     assertEquals(ENTITY1, result.next());
     assertNull(result.next());
@@ -514,8 +517,8 @@ public void testGetArray() {
     assertTrue(entity3.isNull("null"));
     assertFalse(entity3.getBoolean("bool"));
     assertEquals(LIST_VALUE2.get(), entity3.getList("list"));
-    Entity partial1 = entity3.getEntity("partial1");
-    Entity partial2 = entity3.getEntity("partial2");
+    FullEntity partial1 = entity3.getEntity("partial1");
+    FullEntity partial2 = entity3.getEntity("partial2");
     assertEquals(PARTIAL_ENTITY2, partial1);
     assertEquals(ENTITY2, partial2);
     assertEquals(Value.Type.BOOLEAN, entity3.getValue("bool").type());
@@ -533,7 +536,7 @@ public void testGetArray() {
 
   @Test
   public void testAddEntity() {
-    List> keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
+    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -545,7 +548,7 @@ public void testAddEntity() {
       // expected;
     }
 
-    List> entities = datastore.add(ENTITY3, PARTIAL_ENTITY1, PARTIAL_ENTITY2);
+    List entities = datastore.add(ENTITY3, PARTIAL_ENTITY1, PARTIAL_ENTITY2);
     assertEquals(ENTITY3, datastore.get(ENTITY3.key()));
     assertEquals(ENTITY3, entities.get(0));
     assertEquals(PARTIAL_ENTITY1.properties(), entities.get(1).properties());
@@ -559,7 +562,7 @@ public void testAddEntity() {
 
   @Test
   public void testUpdate() {
-    List> keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
+    List keys = datastore.fetch(ENTITY1.key(), ENTITY3.key());
     assertEquals(ENTITY1, keys.get(0));
     assertNull(keys.get(1));
     assertEquals(2, keys.size());
@@ -580,7 +583,7 @@ public void testUpdate() {
 
   @Test
   public void testPut() {
-    Iterator> keys =
+    Iterator keys =
         datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
@@ -599,7 +602,7 @@ public void testPut() {
 
   @Test
   public void testDelete() {
-    Iterator> keys =
+    Iterator keys =
         datastore.fetch(ENTITY1.key(), ENTITY2.key(), ENTITY3.key()).iterator();
     assertEquals(ENTITY1, keys.next());
     assertEquals(ENTITY2, keys.next());
diff --git a/src/test/java/com/google/gcloud/datastore/EntityTest.java b/src/test/java/com/google/gcloud/datastore/EntityTest.java
index f4fc970e7ca9..bb6d1a6eab73 100644
--- a/src/test/java/com/google/gcloud/datastore/EntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/EntityTest.java
@@ -25,27 +25,20 @@ public class EntityTest {
   private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
   private static final Key KEY2 = Key.builder("ds1", "k2", 1).build();
   private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k2").build();
-  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
-  private static final Entity INCOMPLETE_ENTITY =
+  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
+  private static final FullEntity INCOMPLETE_ENTITY =
       Entity.builder(INCOMPLETE_KEY).set("a", "b").build();
 
   @Test
-  public void testGetKey() throws Exception {
+  public void testEntity() throws Exception {
     assertTrue(ENTITY.hasKey());
     assertEquals(KEY1, ENTITY.key());
     assertEquals("bar", ENTITY.getString("foo"));
   }
-  @Test
-  public void testNoKey() throws Exception {
-    Entity entity = Entity.builder().set("foo", "bar").build();
-    assertFalse(entity.hasKey());
-    assertNull(entity.key());
-    assertEquals("bar", entity.getString("foo"));
-  }
 
   @Test
   public void testCopyFrom() throws Exception {
-    Entity.Builder builder = Entity.builder(ENTITY);
+    Entity.Builder builder = Entity.builder(ENTITY);
     assertEquals(ENTITY, builder.build());
     Entity entity = builder.key(KEY2).build();
     assertNotEquals(ENTITY, entity);
@@ -55,8 +48,8 @@ public void testCopyFrom() throws Exception {
 
   @Test
   public void testCopyFromIncompleteEntity() throws Exception {
-    Entity.Builder builder = Entity.builder(KEY2, INCOMPLETE_ENTITY);
-    Entity entity = builder.build();
+    Entity.Builder builder = Entity.builder(KEY2, INCOMPLETE_ENTITY);
+    Entity entity = builder.build();
     assertNotEquals(INCOMPLETE_ENTITY, entity);
     assertEquals(INCOMPLETE_ENTITY.properties(), entity.properties());
   }
diff --git a/src/test/java/com/google/gcloud/datastore/FullEntityTest.java b/src/test/java/com/google/gcloud/datastore/FullEntityTest.java
new file mode 100644
index 000000000000..1d62c7a6dfae
--- /dev/null
+++ b/src/test/java/com/google/gcloud/datastore/FullEntityTest.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+public class FullEntityTest {
+
+  private static final Key KEY1 = Key.builder("ds1", "k1", "n1").build();
+  private static final Key KEY2 = Key.builder("ds1", "k2", 1).build();
+  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k2").build();
+  private static final Entity ENTITY = Entity.builder(KEY1).set("foo", "bar").build();
+  private static final FullEntity COMPLETE_ENTITY1 = ENTITY;
+  private static final FullEntity COMPLETE_ENTITY2 =
+      FullEntity.builder(KEY2).set("foo", "bar").build();
+  private static final FullEntity INCOMPLETE_ENTITY =
+      Entity.builder(INCOMPLETE_KEY).set("a", "b").build();
+
+  @Test
+  public void testFullEntity() throws Exception {
+    assertTrue(COMPLETE_ENTITY1.hasKey());
+    assertEquals(KEY1, COMPLETE_ENTITY1.key());
+    assertEquals("bar", COMPLETE_ENTITY1.getString("foo"));
+
+    assertTrue(COMPLETE_ENTITY2.hasKey());
+    assertEquals(KEY2, COMPLETE_ENTITY2.key());
+    assertEquals("bar", COMPLETE_ENTITY2.getString("foo"));
+  }
+
+  @Test
+  public void testNoKey() throws Exception {
+    FullEntity entity = FullEntity.builder().set("foo", "bar").build();
+    assertFalse(entity.hasKey());
+    assertNull(entity.key());
+    assertEquals("bar", entity.getString("foo"));
+
+    entity = FullEntity.builder((IncompleteKey) null).build();
+    assertFalse(entity.hasKey());
+    assertNull(entity.key());
+  }
+
+  @Test
+  public void testCopyFrom() throws Exception {
+    FullEntity.Builder builder1 = FullEntity.builder(ENTITY);
+    assertEquals(ENTITY, builder1.build());
+
+    builder1 = FullEntity.builder(COMPLETE_ENTITY1);
+    assertEquals(COMPLETE_ENTITY1, builder1.build());
+
+    FullEntity.Builder builder2 = FullEntity.builder(INCOMPLETE_ENTITY);
+    assertEquals(INCOMPLETE_ENTITY, builder2.build());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
index cab1fef6ca84..f67d719af265 100644
--- a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
@@ -32,7 +32,8 @@ public class ProjectionEntityTest {
   private static final DateTimeValue DATE_TIME_VALUE = DateTimeValue.of(DateTime.now());
   private static final LongValue LONG_INDEX_VALUE =
       LongValue.builder(DATE_TIME_VALUE.get().timestampMicroseconds()).meaning(18).build();
-  private static final ProjectionEntity ENTITY1 = new ProjectionEntity.Builder().key(KEY).set("a", "b").build();
+  private static final ProjectionEntity ENTITY1 =
+      new ProjectionEntity.Builder().key(KEY).set("a", "b").build();
   private static final ProjectionEntity ENTITY2 = new ProjectionEntity.Builder()
       .set("a", STRING_INDEX_VALUE)
       .set("b", BLOB_VALUE)
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 63b2f1fc2cdd..103fe07c3310 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -55,13 +55,13 @@ public class SerializationTest {
       .addBinding(20)
       .namespace("ns1")
       .build();
-  private static final Query> GQL2 =
+  private static final Query GQL2 =
       Query.gqlQueryBuilder(Query.Type.FULL, "select * from kind1 where name = @name and age > @1")
       .setBinding("name", "name1")
       .addBinding(20)
       .namespace("ns1")
       .build();
-  private static final Query> QUERY1 =
+  private static final Query QUERY1 =
       Query.fullQueryBuilder().kind("kind1").build();
   private static final Query QUERY2 = Query.keyOnlyQueryBuilder()
       .kind("k")
@@ -89,8 +89,8 @@ public class SerializationTest {
   private static final BlobValue BLOB_VALUE = BlobValue.of(BLOB1);
   private static final RawValue RAW_VALUE = RawValue.of(
       DatastoreV1.Value.newBuilder().setBlobKeyValue("blob-key").setMeaning(18).build());
-  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
-  private static final Entity ENTITY2 =
+  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
+  private static final Entity ENTITY2 =
       Entity.builder(KEY2).set("null", NullValue.of()).build();
   private static final Entity ENTITY3 = Entity.builder(KEY2)
       .set("p1", StringValue.builder("hi1").meaning(10).build())
@@ -98,15 +98,13 @@ public class SerializationTest {
       .set("p3", LongValue.builder(100).indexed(false).meaning(100).build())
       .set("blob", BLOB1)
       .build();
-  private static final Entity EMBEDDED_ENTITY1 = (Entity) ENTITY1;
-  private static final Entity EMBEDDED_ENTITY2 = (Entity) ENTITY2;
-  private static final Entity EMBEDDED_ENTITY3 = Entity.builder(INCOMPLETE_KEY1)
+  private static final FullEntity EMBEDDED_ENTITY = Entity.builder(INCOMPLETE_KEY1)
       .set("p1", STRING_VALUE)
       .set("p2", LongValue.builder(100).indexed(false).meaning(100).build())
       .build();
-  private static final EntityValue EMBEDDED_ENTITY_VALUE1 = EntityValue.of(EMBEDDED_ENTITY1);
-  private static final EntityValue EMBEDDED_ENTITY_VALUE2 = EntityValue.of(EMBEDDED_ENTITY2);
-  private static final EntityValue EMBEDDED_ENTITY_VALUE3 = EntityValue.of(EMBEDDED_ENTITY3);
+  private static final EntityValue EMBEDDED_ENTITY_VALUE1 = EntityValue.of(ENTITY1);
+  private static final EntityValue EMBEDDED_ENTITY_VALUE2 = EntityValue.of(ENTITY2);
+  private static final EntityValue EMBEDDED_ENTITY_VALUE3 = EntityValue.of(EMBEDDED_ENTITY);
   private static final ListValue LIST_VALUE = ListValue.builder()
       .addValue(NULL_VALUE)
       .addValue(STRING_VALUE)
@@ -148,8 +146,8 @@ public void testValues() throws Exception {
   @Test
   public void testTypes() throws Exception {
     Serializable[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
-        ENTITY3, EMBEDDED_ENTITY1, EMBEDDED_ENTITY2, EMBEDDED_ENTITY3, PROJECTION_ENTITY,
-        DATE_TIME1, BLOB1, CURSOR1, GQL1, GQL2, QUERY1, QUERY2, QUERY3};
+        ENTITY3, EMBEDDED_ENTITY, PROJECTION_ENTITY, DATE_TIME1, BLOB1, CURSOR1, GQL1, GQL2,
+        QUERY1, QUERY2, QUERY3};
     for (Serializable obj : types) {
       Object copy = serializeAndDeserialize(obj);
       assertEquals(obj, obj);

From cad5720364b29b10cf06c29ce892a56a1d308641 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sat, 14 Mar 2015 19:45:56 -0700
Subject: [PATCH 133/732] Remove StructuredQuery subclasses and allocateId from
 KeyFactory

---
 .../gcloud/datastore/DatastoreHelper.java     |   4 +
 .../datastore/DatastoreServiceImpl.java       |   2 +-
 .../com/google/gcloud/datastore/GqlQuery.java |  18 +--
 .../google/gcloud/datastore/KeyFactory.java   |  22 +--
 .../com/google/gcloud/datastore/Query.java    |  91 ++++++-------
 .../gcloud/datastore/QueryResultsImpl.java    |  22 +--
 .../gcloud/datastore/StructuredQuery.java     | 125 +++++++-----------
 .../google/gcloud/datastore/package-info.java |   2 +-
 .../gcloud/datastore/DatastoreHelperTest.java |  15 +++
 .../datastore/DatastoreServiceTest.java       |  29 ++--
 .../gcloud/datastore/KeyFactoryTest.java      |  27 +---
 .../gcloud/datastore/SerializationTest.java   |   6 +-
 12 files changed, 153 insertions(+), 210 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index d51147b4d053..0ae0cbea218a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -46,6 +46,10 @@ static Entity add(DatastoreWriter writer, FullEntity entity) {
     return writer.add(new FullEntity[] {entity}).get(0);
   }
 
+  static KeyFactory newKeyFactory(DatastoreServiceOptions options) {
+    return new KeyFactory(options.dataset()).namespace(options.namespace());
+  }
+
   /**
    * Returns a list with a value for each given key (ordered by input).
    * A {@code null} would be returned for non-existing keys.
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 98dcdf0f32da..119a29a38792 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -327,7 +327,7 @@ public void delete(Key... keys) {
 
   @Override
   public KeyFactory newKeyFactory() {
-    return new KeyFactory(this);
+    return DatastoreHelper.newKeyFactory(options());
   }
 
   private DatastoreV1.CommitResponse commitMutation(DatastoreV1.Mutation.Builder mutationPb) {
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 1ebb313cdd26..e7b9275a83f9 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -45,7 +45,7 @@
  * 
When the type of the results is known the preferred usage would be:
  * 
{@code
  *   Query<Entity> query =
- *       Query.gqlQueryBuilder(Query.Type.FULL, "select * from kind").build();
+ *       Query.gqlQueryBuilder(Query.ResultType.ENTITY, "select * from kind").build();
  *   QueryResults<Entity> results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
@@ -159,15 +159,15 @@ static Binding fromPb(DatastoreV1.GqlQueryArg argPb) {
    */
   public static final class Builder {
 
-    private final Type type;
+    private final ResultType resultType;
     private String namespace;
     private String queryString;
     private boolean allowLiteral;
     private final Map namedBindings = new TreeMap<>();
     private final List positionalBindings = new LinkedList<>();
 
-    Builder(Type type, String query) {
-      this.type = checkNotNull(type);
+    Builder(ResultType resultType, String query) {
+      this.resultType = checkNotNull(resultType);
       queryString = checkNotNull(query);
     }
 
@@ -311,7 +311,7 @@ private static Binding toBinding(String name, Value.BuilderFactory builderFactor
   }
 
   private GqlQuery(Builder builder) {
-    super(builder.type, builder.namespace);
+    super(builder.resultType, builder.namespace);
     queryString = builder.queryString;
     allowLiteral = builder.allowLiteral;
     namedBindings = ImmutableList.copyOf(builder.namedBindings.values());
@@ -395,13 +395,13 @@ protected GqlQuery nextQuery(DatastoreV1.QueryResultBatch responsePb) {
   }
 
   @Override
-  protected Object fromPb(Type type, String namespace, byte[] bytesPb)
+  protected Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
-    return fromPb(type, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
+    return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
   }
 
-  private static  GqlQuery fromPb(Type type, String ns, DatastoreV1.GqlQuery queryPb) {
-    Builder builder = new Builder<>(type, queryPb.getQueryString());
+  private static  GqlQuery fromPb(ResultType resultType, String ns, DatastoreV1.GqlQuery queryPb) {
+    Builder builder = new Builder<>(resultType, queryPb.getQueryString());
     builder.namespace(ns);
     if (queryPb.hasAllowLiteral()) {
       builder.allowLiteral = queryPb.getAllowLiteral();
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index 1bf14048f598..a7ce8b1ac8db 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -16,8 +16,6 @@
 
 package com.google.gcloud.datastore;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import com.google.common.collect.ImmutableList;
 
 /**
@@ -26,12 +24,8 @@
  */
 public final class KeyFactory extends BaseKey.Builder {
 
-  private final DatastoreService service;
-
-  public KeyFactory(DatastoreService service) {
-    super(checkNotNull(service).options().dataset());
-    this.service = service;
-    namespace(service.options().namespace());
+  public KeyFactory(String dataset) {
+    super(dataset);
   }
 
   public IncompleteKey newKey() {
@@ -52,20 +46,8 @@ public Key newKey(long id) {
     return new Key(dataset, namespace, path);
   }
 
-  /**
-   * Return a key with a newly allocated id.
-   * @throws DatastoreServiceException if allocation failed.
-   */
-  public Key allocateId() {
-    return service.allocateId(newKey());
-  }
-
   @Override
   protected IncompleteKey build() {
     return newKey();
   }
-
-  DatastoreService datastore() {
-    return service;
-  }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 83b5396cd4c5..9d71b4073cbd 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -22,9 +22,9 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
 import com.google.common.collect.Maps;
-import com.google.gcloud.datastore.StructuredQuery.FullQueryBuilder;
-import com.google.gcloud.datastore.StructuredQuery.KeyOnlyQueryBuilder;
-import com.google.gcloud.datastore.StructuredQuery.ProjectionQueryBuilder;
+import com.google.gcloud.datastore.StructuredQuery.EntityQueryBuilder;
+import com.google.gcloud.datastore.StructuredQuery.KeyQueryBuilder;
+import com.google.gcloud.datastore.StructuredQuery.ProjectionEntityQueryBuilder;
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 
@@ -42,22 +42,22 @@ public abstract class Query extends Serializable {
 
   private static final long serialVersionUID = -2748141759901313101L;
 
-  private final Type type;
+  private final ResultType resultType;
   private final String namespace;
 
   /**
    * This class represents the expected type of the result.
-   *   FULL: A full entity represented by {@link Entity}.
-   *   PROJECTION: A projection entity, represented by {@link ProjectionEntity}.
-   *   KEY_ONLY: An entity's {@link Key}.
+   *   ENTITY: A full entity represented by {@link Entity}.
+   *   PROJECTION_ENTITY: A projection entity, represented by {@link ProjectionEntity}.
+   *   KEY: An entity's {@link Key}.
    */
-  public abstract static class Type implements java.io.Serializable {
+  public abstract static class ResultType implements java.io.Serializable {
 
     private static final long serialVersionUID = 2104157695425806623L;
-    private static final Map>
+    private static final Map>
         PB_TO_INSTANCE = Maps.newEnumMap(DatastoreV1.EntityResult.ResultType.class);
 
-    static final Type UNKNOWN = new Type(null, Object.class) {
+    static final ResultType UNKNOWN = new ResultType(null, Object.class) {
 
       private static final long serialVersionUID = 1602329532153860907L;
 
@@ -72,8 +72,8 @@ public abstract static class Type implements java.io.Serializable {
       }
     };
 
-    public static final Type FULL =
-        new Type(DatastoreV1.EntityResult.ResultType.FULL, Entity.class) {
+    public static final ResultType ENTITY =
+        new ResultType(DatastoreV1.EntityResult.ResultType.FULL, Entity.class) {
 
       private static final long serialVersionUID = 7712959777507168274L;
 
@@ -82,8 +82,8 @@ public abstract static class Type implements java.io.Serializable {
       }
     };
 
-    public static final Type KEY_ONLY =
-        new Type(DatastoreV1.EntityResult.ResultType.KEY_ONLY, Key.class) {
+    public static final ResultType KEY =
+        new ResultType(DatastoreV1.EntityResult.ResultType.KEY_ONLY, Key.class) {
 
       private static final long serialVersionUID = -8514289244104446252L;
 
@@ -92,8 +92,9 @@ public abstract static class Type implements java.io.Serializable {
       }
     };
 
-    public static final Type PROJECTION = new Type(
-        DatastoreV1.EntityResult.ResultType.PROJECTION, ProjectionEntity.class) {
+    public static final ResultType PROJECTION_ENTITY =
+        new ResultType(DatastoreV1.EntityResult.ResultType.PROJECTION,
+            ProjectionEntity.class) {
 
           private static final long serialVersionUID = -7591409419690650246L;
 
@@ -103,14 +104,14 @@ public abstract static class Type implements java.io.Serializable {
     };
 
     private final Class resultClass;
-    private final DatastoreV1.EntityResult.ResultType resultType;
+    private final DatastoreV1.EntityResult.ResultType queryType;
 
     @SuppressWarnings("unchecked")
-    private Type(DatastoreV1.EntityResult.ResultType resultType, Class resultClass) {
-      this.resultType = resultType;
+    private ResultType(DatastoreV1.EntityResult.ResultType queryType, Class resultClass) {
+      this.queryType = queryType;
       this.resultClass = resultClass;
-      if (resultType != null) {
-        PB_TO_INSTANCE.put(resultType, this);
+      if (queryType != null) {
+        PB_TO_INSTANCE.put(queryType, this);
       }
     }
 
@@ -128,39 +129,39 @@ public boolean equals(Object obj) {
       if (obj == this) {
         return true;
       }
-      if (!(obj instanceof Type)) {
+      if (!(obj instanceof ResultType)) {
         return false;
       }
-      Type other = (Type) obj;
+      ResultType other = (ResultType) obj;
       return resultClass.equals(other.resultClass);
     }
 
     @Override
     public String toString() {
       ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
-      toStringHelper.add("resultType", resultType);
+      toStringHelper.add("queryType", queryType);
       toStringHelper.add("resultClass", resultClass);
       return toStringHelper.toString();
     }
 
-    boolean isAssignableFrom(Type otherType) {
-      return resultClass.isAssignableFrom(otherType.resultClass);
+    boolean isAssignableFrom(ResultType otherResultType) {
+      return resultClass.isAssignableFrom(otherResultType.resultClass);
     }
 
     protected abstract V convert(DatastoreV1.Entity entityPb);
 
-    static Type fromPb(DatastoreV1.EntityResult.ResultType typePb) {
+    static ResultType fromPb(DatastoreV1.EntityResult.ResultType typePb) {
       return MoreObjects.firstNonNull(PB_TO_INSTANCE.get(typePb), UNKNOWN);
     }
   }
 
-  Query(Type type, String namespace) {
-    this.type = checkNotNull(type);
+  Query(ResultType resultType, String namespace) {
+    this.resultType = checkNotNull(resultType);
     this.namespace = namespace;
   }
 
-  Type type() {
-    return type;
+  ResultType type() {
+    return resultType;
   }
 
   public String namespace() {
@@ -170,7 +171,7 @@ public String namespace() {
   @Override
   public String toString() {
     ToStringHelper toStringHelper = MoreObjects.toStringHelper(this);
-    toStringHelper.add("type", type);
+    toStringHelper.add("type", resultType);
     toStringHelper.add("namespace", namespace);
     toStringHelper.add("queryPb", super.toString());
     return toStringHelper.toString();
@@ -178,10 +179,10 @@ public String toString() {
 
   @Override
   protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    return fromPb(type, namespace, bytesPb);
+    return fromPb(resultType, namespace, bytesPb);
   }
 
-  protected abstract Object fromPb(Type type, String namespace, byte[] bytesPb)
+  protected abstract Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException;
 
   protected abstract void populatePb(DatastoreV1.RunQueryRequest.Builder requestPb);
@@ -194,7 +195,7 @@ protected abstract Object fromPb(Type type, String namespace, byte[] bytesPb)
    * @see GQL Reference
    */
   public static GqlQuery.Builder gqlQueryBuilder(String gql) {
-    return gqlQueryBuilder(Type.UNKNOWN, gql);
+    return gqlQueryBuilder(ResultType.UNKNOWN, gql);
   }
 
   /**
@@ -202,28 +203,28 @@ public static GqlQuery.Builder gqlQueryBuilder(String gql) {
    *
    * @see GQL Reference
    */
-  public static  GqlQuery.Builder gqlQueryBuilder(Type type, String gql) {
-    return new GqlQuery.Builder<>(type, gql);
+  public static  GqlQuery.Builder gqlQueryBuilder(ResultType resultType, String gql) {
+    return new GqlQuery.Builder<>(resultType, gql);
   }
 
   /**
-   * Returns a new {@link StructuredQuery} builder for full queries.
+   * Returns a new {@link StructuredQuery} builder for full (complete entities) queries.
    */
-  public static FullQueryBuilder fullQueryBuilder() {
-    return new FullQueryBuilder();
+  public static EntityQueryBuilder entityQueryBuilder() {
+    return new EntityQueryBuilder();
   }
 
   /**
-   * Returns a new {@link StructuredQuery} builder for key-only queries.
+   * Returns a new {@link StructuredQuery} builder for key only queries.
    */
-  public static KeyOnlyQueryBuilder keyOnlyQueryBuilder() {
-    return new KeyOnlyQueryBuilder();
+  public static KeyQueryBuilder keyQueryBuilder() {
+    return new KeyQueryBuilder();
   }
 
   /**
    * Returns a new {@link StructuredQuery} builder for projection queries.
    */
-  public static ProjectionQueryBuilder projectionQueryBuilder() {
-    return new ProjectionQueryBuilder();
+  public static ProjectionEntityQueryBuilder projectionQueryBuilder() {
+    return new ProjectionEntityQueryBuilder();
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
index 324a63d7a2a5..fa88722bc8b6 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
@@ -20,7 +20,7 @@
 import com.google.api.services.datastore.DatastoreV1.QueryResultBatch.MoreResultsType;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.AbstractIterator;
-import com.google.gcloud.datastore.Query.Type;
+import com.google.gcloud.datastore.Query.ResultType;
 
 import java.util.Iterator;
 import java.util.Objects;
@@ -30,9 +30,9 @@ class QueryResultsImpl extends AbstractIterator implements QueryResults
   private final DatastoreServiceImpl datastore;
   private final DatastoreV1.ReadOptions readOptionsPb;
   private final DatastoreV1.PartitionId partitionIdPb;
-  private final Query.Type queryType;
+  private final ResultType queryResultType;
   private Query query;
-  private Query.Type actualType;
+  private ResultType actualResultType;
   private DatastoreV1.QueryResultBatch queryResultBatchPb;
   private boolean lastBatch;
   private Iterator entityResultPbIter;
@@ -44,7 +44,7 @@ class QueryResultsImpl extends AbstractIterator implements QueryResults
     this.datastore = datastore;
     this.readOptionsPb = readOptionsPb;
     this.query = query;
-    queryType = query.type();
+    queryResultType = query.type();
     DatastoreV1.PartitionId.Builder pbBuilder = DatastoreV1.PartitionId.newBuilder();
     pbBuilder.setDatasetId(datastore.options().dataset());
     if (query.namespace() != null) {
@@ -67,13 +67,13 @@ private void sendRequest() {
     lastBatch = queryResultBatchPb.getMoreResults() != MoreResultsType.NOT_FINISHED;
     entityResultPbIter = queryResultBatchPb.getEntityResultList().iterator();
     // cursor = resultPb.getSkippedCursor(); // available in v1beta3, use startCursor if not skipped
-    actualType = Type.fromPb(queryResultBatchPb.getEntityResultType());
-    if (Objects.equals(queryType, Type.PROJECTION)) {
+    actualResultType = ResultType.fromPb(queryResultBatchPb.getEntityResultType());
+    if (Objects.equals(queryResultType, ResultType.PROJECTION_ENTITY)) {
       // projection entity can represent all type of results
-      actualType = Type.PROJECTION;
+      actualResultType = ResultType.PROJECTION_ENTITY;
     }
-    Preconditions.checkState(queryType.isAssignableFrom(actualType),
-        "Unexpected result type " + actualType + " vs " + queryType);
+    Preconditions.checkState(queryResultType.isAssignableFrom(actualResultType),
+        "Unexpected result type " + actualResultType + " vs " + queryResultType);
   }
 
   @SuppressWarnings("unchecked")
@@ -88,12 +88,12 @@ protected T computeNext() {
     }
     DatastoreV1.EntityResult entityResultPb = entityResultPbIter.next();
     //cursor = entityResultPb.getCursor(); // only available in v1beta3
-    return (T) actualType.convert(entityResultPb.getEntity());
+    return (T) actualResultType.convert(entityResultPb.getEntity());
   }
 
   @Override
   public Class resultClass() {
-    return actualType.resultClass();
+    return actualResultType.resultClass();
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 73cc7edde09b..fda9fd81097d 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -47,8 +47,8 @@
  *
  * 
A simple query that returns all entities for a specific kind
  * 
{@code
- *   FullQuery query = Query.fullQueryBuilder().kind(kind).build();
- *   QueryResults results = datastore.run(query);
+ *   Query<Entity> query = Query.entityQueryBuilder().kind(kind).build();
+ *   QueryResults<Entity> results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -57,15 +57,15 @@
  *
  * 
A simple key-only query of all entities for a specific kind
  * 
{@code
- *   KeyOnlyQuery keyOnlyQuery =  Query.keyOnlyQueryBuilder().kind(KIND1).build();
- *   QueryResults results = datastore.run(keyOnlyQuery);
+ *   Query<Key> keyOnlyQuery =  Query.keyQueryBuilder().kind(KIND1).build();
+ *   QueryResults<Key> results = datastore.run(keyOnlyQuery);
  *   ...
  * } 

  *
  * 
A less trivial example of a projection query that returns the first 10 results
  * of "age" and "name" properties (sorted and grouped by "age") with an age greater than 18
  * 
{@code
- *   ProjectionQuery query = Query.projectionQueryBuilder()
+ *   Query<ProjectionEntity> query = Query.projectionQueryBuilder()
  *       .kind(kind)
  *       .projection(Projection.property("age"), Projection.first("name"))
  *       .filter(PropertyFilter.gt("age", 18))
@@ -601,7 +601,7 @@ public static Projection first(String property) {
 
   static class BaseBuilder> {
 
-    private final Type type;
+    private final ResultType resultType;
     private String namespace;
     private String kind;
     private final List projection = new LinkedList<>();
@@ -613,8 +613,8 @@ static class BaseBuilder> {
     private int offset;
     private Integer limit;
 
-    BaseBuilder(Type type) {
-      this.type = type;
+    BaseBuilder(ResultType resultType) {
+      this.resultType = resultType;
     }
 
     @SuppressWarnings("unchecked")
@@ -748,32 +748,32 @@ public StructuredQuery build() {
 
   static final class Builder extends BaseBuilder> {
 
-    Builder(Type type) {
-      super(type);
+    Builder(ResultType resultType) {
+      super(resultType);
     }
   }
 
-  public static final class FullQueryBuilder extends BaseBuilder {
+  public static final class EntityQueryBuilder extends BaseBuilder {
 
-    FullQueryBuilder() {
-      super(Type.FULL);
+    EntityQueryBuilder() {
+      super(ResultType.ENTITY);
     }
 
     @Override
-    public FullQuery build() {
-      return new FullQuery(this);
+    public StructuredQuery build() {
+      return new StructuredQuery<>(this);
     }
   }
 
-  public static final class KeyOnlyQueryBuilder extends BaseBuilder {
+  public static final class KeyQueryBuilder extends BaseBuilder {
 
-    KeyOnlyQueryBuilder() {
-      super(Type.KEY_ONLY);
+    KeyQueryBuilder() {
+      super(ResultType.KEY);
       projection(Projection.property(KEY_PROPERTY_NAME));
     }
 
     @Override
-    protected KeyOnlyQueryBuilder mergeFrom(DatastoreV1.Query queryPb) {
+    protected KeyQueryBuilder mergeFrom(DatastoreV1.Query queryPb) {
       super.mergeFrom(queryPb);
       projection(Projection.property(KEY_PROPERTY_NAME));
       clearGroupBy();
@@ -781,91 +781,56 @@ protected KeyOnlyQueryBuilder mergeFrom(DatastoreV1.Query queryPb) {
     }
 
     @Override
-    public KeyOnlyQuery build() {
-      return new KeyOnlyQuery(this);
+    public StructuredQuery build() {
+      return new StructuredQuery<>(this);
     }
   }
 
-  public static final class ProjectionQueryBuilder
-      extends BaseBuilder {
+  public static final class ProjectionEntityQueryBuilder
+      extends BaseBuilder {
 
-    ProjectionQueryBuilder() {
-      super(Type.PROJECTION);
+    ProjectionEntityQueryBuilder() {
+      super(ResultType.PROJECTION_ENTITY);
     }
 
     @Override
-    public ProjectionQuery build() {
-      return new ProjectionQuery(this);
+    public StructuredQuery build() {
+      return new StructuredQuery<>(this);
     }
 
     @Override
-    public ProjectionQueryBuilder clearProjection() {
+    public ProjectionEntityQueryBuilder clearProjection() {
       return super.clearProjection();
     }
 
     @Override
-    public ProjectionQueryBuilder projection(Projection projection, Projection... others) {
+    public ProjectionEntityQueryBuilder projection(Projection projection, Projection... others) {
       return super.projection(projection, others);
     }
 
     @Override
-    public ProjectionQueryBuilder addProjection(Projection projection, Projection... others) {
+    public ProjectionEntityQueryBuilder addProjection(Projection projection, Projection... others) {
       return super.addProjection(projection, others);
     }
 
     @Override
-    public ProjectionQueryBuilder clearGroupBy() {
+    public ProjectionEntityQueryBuilder clearGroupBy() {
       return super.clearGroupBy();
     }
 
     @Override
-    public ProjectionQueryBuilder groupBy(String property, String... others) {
+    public ProjectionEntityQueryBuilder groupBy(String property, String... others) {
       return super.groupBy(property, others);
     }
 
     @Override
-    public ProjectionQueryBuilder addGroupBy(String property, String... others) {
+    public ProjectionEntityQueryBuilder addGroupBy(String property, String... others) {
       return super.addGroupBy(property, others);
     }
   }
 
-  public static final class FullQuery extends StructuredQuery {
-
-    FullQuery(BaseBuilder builder) {
-      super(builder);
-    }
-  }
-
-  public static final class KeyOnlyQuery extends StructuredQuery {
-
-    private static final long serialVersionUID = -7502917784216095473L;
-
-    KeyOnlyQuery(KeyOnlyQueryBuilder builder) {
-      super(builder);
-    }
-  }
-
-  public static final class ProjectionQuery extends StructuredQuery {
-
-    private static final long serialVersionUID = -3333183044486150649L;
-
-    ProjectionQuery(ProjectionQueryBuilder builder) {
-      super(builder);
-    }
-
-    @Override
-    public List projection() {
-      return super.projection();
-    }
-
-    @Override
-    public List groupBy() {
-      return super.groupBy();
-    }
-  }
-
   StructuredQuery(BaseBuilder builder) {
-    super(builder.type, builder.namespace);
+    super(builder.resultType, builder.namespace);
     kind = builder.kind;
     projection = ImmutableList.copyOf(builder.projection);
     filter = builder.filter;
@@ -909,11 +874,11 @@ public String kind() {
     return kind;
   }
 
-  protected boolean keyOnly() {
+  boolean keyOnly() {
     return projection.size() == 1 && KEY_PROPERTY_NAME.equals(projection.get(0).property);
   }
 
-  List projection() {
+  public List projection() {
     return projection;
   }
 
@@ -921,7 +886,7 @@ public Filter filter() {
     return filter;
   }
 
-  List groupBy() {
+  public List groupBy() {
     return groupBy;
   }
 
@@ -1000,20 +965,20 @@ protected DatastoreV1.Query toPb() {
   }
 
   @Override
-  protected Object fromPb(Type type, String namespace, byte[] bytesPb)
+  protected Object fromPb(ResultType resultType, String namespace, byte[] bytesPb)
       throws InvalidProtocolBufferException {
-    return fromPb(type, namespace, DatastoreV1.Query.parseFrom(bytesPb));
+    return fromPb(resultType, namespace, DatastoreV1.Query.parseFrom(bytesPb));
   }
 
-  private static StructuredQuery fromPb(Type type, String namespace,
+  private static StructuredQuery fromPb(ResultType resultType, String namespace,
       DatastoreV1.Query queryPb) {
     BaseBuilder builder;
-    if (type.equals(Type.FULL)) {
-      builder = new FullQueryBuilder();
-    } else if (type.equals(Type.KEY_ONLY)) {
-      builder = new KeyOnlyQueryBuilder();
+    if (resultType.equals(ResultType.ENTITY)) {
+      builder = new EntityQueryBuilder();
+    } else if (resultType.equals(ResultType.KEY)) {
+      builder = new KeyQueryBuilder();
     } else {
-      builder = new ProjectionQueryBuilder();
+      builder = new ProjectionEntityQueryBuilder();
     }
     return builder.namespace(namespace).mergeFrom(queryPb).build();
   }
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index 7661deebb82c..06073f6f2329 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -21,7 +21,7 @@
  * 
 {@code
  * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
  * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
- * KeyFactory keyFactory = new KeyFactory(datastore).kind(kind);
+ * KeyFactory keyFactory = datastore.newKeyFactory().kind(kind);
  * Key key = keyFactory.newKey(keyName);
  * Entity entity = datastore.get(key);
  * if (entity == null) {
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 57089b0a4f2d..cfe6380ce862 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -31,6 +31,21 @@
 
 public class DatastoreHelperTest {
 
+  @Test
+  public void testNewKeyFactory() {
+    DatastoreServiceOptions options = createMock(DatastoreServiceOptions.class);
+    expect(options.dataset()).andReturn("ds1").once();
+    expect(options.namespace()).andReturn("ns1").once();
+    replay(options);
+    KeyFactory keyFactory = DatastoreHelper.newKeyFactory(options);
+    Key key = keyFactory.kind("k").newKey("bla");
+    assertEquals("ds1", key.dataset());
+    assertEquals("ns1", key.namespace());
+    assertEquals("k", key.kind());
+    assertEquals("bla", key.name());
+    verify(options);
+  }
+
   @Test
   public void testAllocateId() throws Exception {
     DatastoreService datastoreService = createStrictMock(DatastoreService.class);
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 25c668f01db8..c8be85a7c7ce 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -26,8 +26,7 @@
 import static org.junit.Assert.fail;
 
 import com.google.common.collect.Iterators;
-import com.google.gcloud.datastore.Query.Type;
-import com.google.gcloud.datastore.StructuredQuery.FullQuery;
+import com.google.gcloud.datastore.Query.ResultType;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
@@ -108,7 +107,7 @@ public void setUp() throws IOException, InterruptedException {
         .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
     datastore = DatastoreServiceFactory.getDefault(options);
-    StructuredQuery query = Query.keyOnlyQueryBuilder().build();
+    StructuredQuery query = Query.keyQueryBuilder().build();
     QueryResults result = datastore.run(query);
     datastore.delete(Iterators.toArray(result, Key.class));
     datastore.add(ENTITY1, ENTITY2);
@@ -184,7 +183,7 @@ public void testTransactionWithRead() {
 
   @Test
   public void testTransactionWithQuery() {
-    Query query = Query.fullQueryBuilder()
+    Query query = Query.entityQueryBuilder()
         .kind(KIND2)
         .filter(PropertyFilter.hasAncestor(KEY2))
         .build();
@@ -327,7 +326,7 @@ public void testNewBatch() {
 
   @Test
   public void testRunGqlQueryNoCasting() {
-    Query query1 = Query.gqlQueryBuilder(Type.FULL, "select * from " + KIND1).build();
+    Query query1 = Query.gqlQueryBuilder(ResultType.ENTITY, "select * from " + KIND1).build();
     QueryResults results1 = datastore.run(query1);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
@@ -335,7 +334,7 @@ public void testRunGqlQueryNoCasting() {
 
     datastore.put(ENTITY3);
     Query query2 =  Query.gqlQueryBuilder(
-        Type.FULL, "select * from " + KIND2 + " order by __key__").build();
+        ResultType.ENTITY, "select * from " + KIND2 + " order by __key__").build();
     QueryResults results2 = datastore.run(query2);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY2, results2.next());
@@ -343,19 +342,19 @@ public void testRunGqlQueryNoCasting() {
     assertEquals(ENTITY3, results2.next());
     assertFalse(results2.hasNext());
 
-    query1 = Query.gqlQueryBuilder(Type.FULL, "select * from bla").build();
+    query1 = Query.gqlQueryBuilder(ResultType.ENTITY, "select * from bla").build();
     results1 = datastore.run(query1);
     assertFalse(results1.hasNext());
 
     Query keyOnlyQuery =
-        Query.gqlQueryBuilder(Type.KEY_ONLY, "select __key__ from " + KIND1).build();
+        Query.gqlQueryBuilder(ResultType.KEY, "select __key__ from " + KIND1).build();
     QueryResults keyOnlyResults = datastore.run(keyOnlyQuery);
     assertTrue(keyOnlyResults.hasNext());
     assertEquals(KEY1, keyOnlyResults.next());
     assertFalse(keyOnlyResults.hasNext());
 
     GqlQuery keyProjectionQuery = Query.gqlQueryBuilder(
-        Type.PROJECTION, "select __key__ from " + KIND1).build();
+        ResultType.PROJECTION_ENTITY, "select __key__ from " + KIND1).build();
     QueryResults keyProjectionResult = datastore.run(keyProjectionQuery);
     assertTrue(keyProjectionResult.hasNext());
     ProjectionEntity projectionEntity = keyProjectionResult.next();
@@ -364,7 +363,7 @@ public void testRunGqlQueryNoCasting() {
     assertFalse(keyProjectionResult.hasNext());
 
     GqlQuery projectionQuery = Query.gqlQueryBuilder(
-        Type.PROJECTION, "select str, date from " + KIND1).build();
+        ResultType.PROJECTION_ENTITY, "select str, date from " + KIND1).build();
 
     QueryResults projectionResult = datastore.run(projectionQuery);
     assertTrue(projectionResult.hasNext());
@@ -399,14 +398,14 @@ public void testRunGqlQueryWithCasting() {
 
   @Test
   public void testRunStructuredQuery() {
-    FullQuery query =
-        Query.fullQueryBuilder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
+    Query query =
+        Query.entityQueryBuilder().kind(KIND1).orderBy(OrderBy.asc("__key__")).build();
     QueryResults results1 = datastore.run(query);
     assertTrue(results1.hasNext());
     assertEquals(ENTITY1, results1.next());
     assertFalse(results1.hasNext());
 
-    Query keyOnlyQuery =  Query.keyOnlyQueryBuilder().kind(KIND1).build();
+    Query keyOnlyQuery =  Query.keyQueryBuilder().kind(KIND1).build();
     QueryResults results2 = datastore.run(keyOnlyQuery);
     assertTrue(results2.hasNext());
     assertEquals(ENTITY1.key(), results2.next());
@@ -446,7 +445,7 @@ public void testRunStructuredQuery() {
   public void testAllocateId() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     IncompleteKey pk1 = keyFactory.newKey();
-    Key key1 = keyFactory.allocateId();
+    Key key1 = datastore.allocateId(pk1);
     assertEquals(key1.dataset(), pk1.dataset());
     assertEquals(key1.namespace(), pk1.namespace());
     assertEquals(key1.ancestors(), pk1.ancestors());
@@ -621,7 +620,7 @@ public void testKeyFactory() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     assertEquals(INCOMPLETE_KEY1, keyFactory.newKey());
     assertEquals(IncompleteKey.builder(INCOMPLETE_KEY1).kind(KIND2).build(),
-        new KeyFactory(datastore).kind(KIND2).newKey());
+        datastore.newKeyFactory().kind(KIND2).newKey());
     assertEquals(KEY1, keyFactory.newKey("name"));
     assertEquals(Key.builder(KEY1).id(2).build(), keyFactory.newKey(2));
   }
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index 4de33197816b..943322ad4a53 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -18,10 +18,6 @@
 
 import static junit.framework.TestCase.assertEquals;
 
-import com.google.api.services.datastore.DatastoreV1;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
-
-import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -32,15 +28,10 @@ public class KeyFactoryTest {
   private static final String DATASET = "dataset";
 
   private KeyFactory keyFactory;
-  private DatastoreRpc mock;
 
   @Before
   public void setUp() {
-    mock = EasyMock.createMock(DatastoreRpc.class);
-    DatastoreServiceOptions options = DatastoreServiceOptions.builder().normalizeDataset(false)
-        .datastoreRpc(mock).dataset(DATASET).build();
-    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
-    keyFactory = new KeyFactory(datastore).kind("k");
+    keyFactory = new KeyFactory(DATASET).kind("k");
   }
 
   @Test
@@ -67,7 +58,7 @@ public void testNewIncompleteKey() throws Exception {
 
   @Test(expected = NullPointerException.class)
   public void testNewIncompleteWithNoKind() {
-    new KeyFactory(keyFactory.datastore()).build();
+    new KeyFactory(DATASET).build();
   }
 
   private void verifyKey(Key key, String name, String namespace, PathElement... ancestors) {
@@ -90,18 +81,4 @@ private void verifyIncompleteKey(IncompleteKey key, String namespace, PathElemen
       assertEquals(ancestor, iter.next());
     }
   }
-
-  @Test
-  public void testAllocateId() throws Exception {
-    IncompleteKey pk = keyFactory.newKey();
-    Key key = keyFactory.newKey(1);
-    DatastoreV1.AllocateIdsRequest.Builder requestPb = DatastoreV1.AllocateIdsRequest.newBuilder();
-    requestPb.addKey(pk.toPb());
-    DatastoreV1.AllocateIdsResponse.Builder responsePb = DatastoreV1.AllocateIdsResponse.newBuilder();
-    responsePb.addKey(key.toPb());
-    EasyMock.expect(mock.allocateIds(requestPb.build())).andReturn(responsePb.build());
-    EasyMock.replay(mock);
-    assertEquals(key, keyFactory.allocateId());
-    EasyMock.verify(mock);
-  }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 103fe07c3310..ef50270bcc60 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -56,14 +56,14 @@ public class SerializationTest {
       .namespace("ns1")
       .build();
   private static final Query GQL2 =
-      Query.gqlQueryBuilder(Query.Type.FULL, "select * from kind1 where name = @name and age > @1")
+      Query.gqlQueryBuilder(Query.ResultType.ENTITY, "select * from kind1 where name = @name and age > @1")
       .setBinding("name", "name1")
       .addBinding(20)
       .namespace("ns1")
       .build();
   private static final Query QUERY1 =
-      Query.fullQueryBuilder().kind("kind1").build();
-  private static final Query QUERY2 = Query.keyOnlyQueryBuilder()
+      Query.entityQueryBuilder().kind("kind1").build();
+  private static final Query QUERY2 = Query.keyQueryBuilder()
       .kind("k")
       .filter(PropertyFilter.eq("p1", "hello"))
       .build();

From ebdf778bc2512cace9b9244d754034384d877309 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 16 Mar 2015 13:30:56 -0700
Subject: [PATCH 134/732] rename projectQueryBuilder method to
 projectionEntityQueryBuilder for consistency

---
 pom.xml                                                       | 2 +-
 src/main/java/com/google/gcloud/datastore/Query.java          | 2 +-
 .../java/com/google/gcloud/datastore/StructuredQuery.java     | 2 +-
 .../com/google/gcloud/datastore/DatastoreServiceTest.java     | 4 ++--
 .../java/com/google/gcloud/datastore/SerializationTest.java   | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9a112e25c12f..724e0895889a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   com.google.gcloud
   gcloud-java
   jar
-  0.0.1
+  0.0.2
   GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/src/main/java/com/google/gcloud/datastore/Query.java
index 9d71b4073cbd..093dc7283327 100644
--- a/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/src/main/java/com/google/gcloud/datastore/Query.java
@@ -224,7 +224,7 @@ public static KeyQueryBuilder keyQueryBuilder() {
   /**
    * Returns a new {@link StructuredQuery} builder for projection queries.
    */
-  public static ProjectionEntityQueryBuilder projectionQueryBuilder() {
+  public static ProjectionEntityQueryBuilder projectionEntityQueryBuilder() {
     return new ProjectionEntityQueryBuilder();
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index fda9fd81097d..deab06944704 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -65,7 +65,7 @@
  * 
A less trivial example of a projection query that returns the first 10 results
  * of "age" and "name" properties (sorted and grouped by "age") with an age greater than 18
  * 
{@code
- *   Query<ProjectionEntity> query = Query.projectionQueryBuilder()
+ *   Query<ProjectionEntity> query = Query.projectionEntityQueryBuilder()
  *       .kind(kind)
  *       .projection(Projection.property("age"), Projection.first("name"))
  *       .filter(PropertyFilter.gt("age", 18))
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index c8be85a7c7ce..4234dd850044 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -412,7 +412,7 @@ public void testRunStructuredQuery() {
     assertFalse(results2.hasNext());
 
     StructuredQuery keyOnlyProjectionQuery =
-        Query.projectionQueryBuilder()
+        Query.projectionEntityQueryBuilder()
         .kind(KIND1).projection(Projection.property("__key__")).build();
     QueryResults results3 = datastore.run(keyOnlyProjectionQuery);
     assertTrue(results3.hasNext());
@@ -421,7 +421,7 @@ public void testRunStructuredQuery() {
     assertTrue(projectionEntity.names().isEmpty());
     assertFalse(results2.hasNext());
 
-    StructuredQuery projectionQuery = Query.projectionQueryBuilder()
+    StructuredQuery projectionQuery = Query.projectionEntityQueryBuilder()
         .kind(KIND2)
         .projection(Projection.property("age"), Projection.first("name"))
         .filter(PropertyFilter.gt("age", 18))
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index ef50270bcc60..3260570a9a3b 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -67,7 +67,7 @@ public class SerializationTest {
       .kind("k")
       .filter(PropertyFilter.eq("p1", "hello"))
       .build();
-  private static final Query QUERY3 = Query.projectionQueryBuilder()
+  private static final Query QUERY3 = Query.projectionEntityQueryBuilder()
       .kind("k")
       .namespace("ns1")
       .projection(Projection.property("p"))

From f22030bfd7ca7329ad86775067c09a0b91b74132 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 17 Mar 2015 17:41:00 -0700
Subject: [PATCH 135/732] wip

---
 .../java/com/google/gcloud/storage/Acl.java   | 90 +++++++++++++------
 .../google/gcloud/storage/ListOptions.java    |  1 +
 .../google/gcloud/storage/StorageService.java |  2 +-
 3 files changed, 66 insertions(+), 27 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/ListOptions.java

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 7cd251d5b43d..15d9444311d5 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -38,63 +38,109 @@ public enum Type {
     PROJECT
   }
 
-  public static class Builder {
-    private Builder() {
+  public static class Domain extends Acl {
+
+    private static final long serialVersionUID = -3033025857280447253L;
+    private final String domain;
+
+    Domain(String domain, Role role) {
+      super(Type.DOMAIN, role);
+      this.domain = domain;
+    }
 
+    public String domain() {
+      return domain;
     }
 
-    Acl build()
+    public static Domain of(String domain, Role role) {
+      return new Domain(domain, role);
+    }
   }
 
-  public static class Domain extends Acl {
+  public static class Group extends Acl {
 
-    private final String domain;
+    private static final long serialVersionUID = -1660987136294408826L;
+    private final String email;
 
-    Domain(Role role, String domain) {
-      super(Type.USER, role);
+    Group(String email, Role role) {
+      super(Type.GROUP, role);
       this.email = email;
     }
 
-    public static User domain(Role role, String domain) {
-      return new User(role, email);
+    public String email() {
+      return email;
+    }
+
+    public static Group of(String email, Role role) {
+      return new Group(email, role);
     }
   }
 
   public static class User extends Acl {
 
     private static final long serialVersionUID = 3076518036392737008L;
+    private static final String ALL_USERS = "allUsers";
+    private static final String ALL_AUTHENTICATED_USERS = "allAuthenticatedUsers";
 
     private final String email;
 
-    User(Role role, String email) {
+    User(String email, Role role) {
       super(Type.USER, role);
       this.email = email;
     }
 
-    String email() {
+    public String email() {
       return email;
     }
 
-    public static User email(Role role, String email) {
-      return new User(role, email);
+    public static User of(String email, Role role) {
+      return new User(email, role);
     }
 
-    public static User allUsers(Role role) {
-      return email(role, "allUsers");
+    public static User ofAllUsers(Role role) {
+      return of(ALL_USERS, role);
     }
 
-    public static User allAuthenticatedUsers(Role role) {
-      return email(role, "allAuthenticatedUsers");
+    public static User ofAllAuthenticatedUsers(Role role) {
+      return of(ALL_AUTHENTICATED_USERS, role);
     }
   }
 
+  public static class Project extends Acl {
+
+    private final String projectId;
+    private final ProjectRole pRole;
+
+    enum ProjectRole {
+      OWNERS,
+      EDITORS,
+      VIEWERS
+    }
+
+    Project(ProjectRole pRole, String projectId, Role role) {
+      super(Type.PROJECT, role);
+      this.pRole = pRole;
+      this.projectId = projectId;
+    }
+
+    public ProjectRole projectRole() {
+      return pRole;
+    }
+
+    public String projectId() {
+      return projectId;
+    }
+
+    public static Project of(ProjectRole pRole, String projectId, Role role) {
+      return new Project(pRole, projectId, role);
+    }
+  }
 
   Acl(Type type, Role role) {
     this.type = type;
     this.role = role;
   }
 
-
   public Type type() {
     return type;
   }
@@ -102,12 +148,4 @@ public Type type() {
   public Role role() {
     return role;
   }
-
-  public Builder toBuilder() {
-
-  }
-
-  public static Builder builder() {
-    return new Builder();
-  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java
new file mode 100644
index 000000000000..9ca3d9120ff1
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/ListOptions.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private final boolean recurse;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;

  public static class Builder {

    private boolean recurse;
    private String prefix;
    private String cursor;
    private int maxResults;

    public Builder() {
    }

    public Builder recurse(boolean recurse) {
      this.recurse = recurse;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recurse = builder.recurse;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
  }

  public boolean recurse() {
    return recurse;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 77147b3621ea..48947537653b 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -24,7 +24,7 @@ public interface StorageService extends Service {
 
   Iterable list();
 
-  Iterable list(ListSettings settings);
+  Iterable list(ListOptions settings);
 
   BucketInfo get(String bucket);
 

From 65b21ade01220a72b3b02ce1e9ec6271a20184bb Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 18 Mar 2015 15:25:28 -0700
Subject: [PATCH 136/732] Providing a default app credentials

---
 pom.xml                                       | 10 ++++
 .../{AuthConfig.java => AuthCredentials.java} | 54 ++++++++++++++-----
 .../com/google/gcloud/ServiceOptions.java     | 39 ++++++++------
 .../com/google/gcloud/spi/DatastoreRpc.java   |  1 -
 .../gcloud/spi/DefaultDatastoreRpc.java       |  1 -
 .../google/gcloud/spi/DefaultStorageRpc.java  |  1 -
 .../google/gcloud/spi/ServiceRpcProvider.java |  1 -
 .../google/gcloud/datastore/BaseEntity.java   |  1 +
 .../datastore/DatastoreServiceException.java  |  4 +-
 .../datastore/DatastoreServiceImpl.java       |  4 +-
 .../datastore/DatastoreServiceOptions.java    |  6 +--
 .../google/gcloud/datastore/FullEntity.java   |  2 +-
 .../com/google/gcloud/spi/DatastoreRpc.java   |  1 +
 .../gcloud => }/spi/DatastoreRpcFactory.java  |  2 +-
 .../gcloud/spi/DefaultDatastoreRpc.java       |  1 +
 .../google/gcloud/spi/DefaultStorageRpc.java  |  1 +
 .../gcloud => }/spi/ServiceRpcFactory.java    |  2 +-
 .../google/gcloud/spi/ServiceRpcProvider.java |  1 +
 .../google/gcloud => }/spi/StorageRpc.java    |  2 +-
 .../gcloud => }/spi/StorageRpcFactory.java    |  2 +-
 .../gcloud/storage/StorageServiceImpl.java    |  2 +-
 .../gcloud/storage/StorageServiceOptions.java |  4 +-
 .../DatastoreServiceExceptionTest.java        |  2 +-
 .../DatastoreServiceOptionsTest.java          |  2 +-
 24 files changed, 96 insertions(+), 50 deletions(-)
 rename src/main/java/com/google/gcloud/{AuthConfig.java => AuthCredentials.java} (62%)
 delete mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
 delete mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
 delete mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
 delete mode 100644 src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
 create mode 100644 src/main/java/com/google/gcloud/spi/DatastoreRpc.java
 rename src/main/java/com/google/gcloud/{com/google/gcloud => }/spi/DatastoreRpcFactory.java (76%)
 create mode 100644 src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
 create mode 100644 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
 rename src/main/java/com/google/gcloud/{com/google/gcloud => }/spi/ServiceRpcFactory.java (79%)
 create mode 100644 src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
 rename src/main/java/com/google/gcloud/{com/google/gcloud => }/spi/StorageRpc.java (71%)
 rename src/main/java/com/google/gcloud/{com/google/gcloud => }/spi/StorageRpcFactory.java (77%)

diff --git a/pom.xml b/pom.xml
index 724e0895889a..e144308fd875 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,6 +49,16 @@
     
   
   
+    
+        com.google.auth
+        google-auth-library-credentials
+        0.1.0
+    
+    
+        com.google.auth
+        google-auth-library-oauth2-http
+        0.1.0
+    
     
       com.google.http-client
       google-http-client
diff --git a/src/main/java/com/google/gcloud/AuthConfig.java b/src/main/java/com/google/gcloud/AuthCredentials.java
similarity index 62%
rename from src/main/java/com/google/gcloud/AuthConfig.java
rename to src/main/java/com/google/gcloud/AuthCredentials.java
index c2accdf438f8..0514c26a4e7e 100644
--- a/src/main/java/com/google/gcloud/AuthConfig.java
+++ b/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -26,15 +26,20 @@
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.jackson.JacksonFactory;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
 
 import java.io.IOException;
 import java.security.GeneralSecurityException;
 import java.security.PrivateKey;
 import java.util.Set;
 
-public abstract class AuthConfig {
+/**
+ * Credentials for accessing Google Cloud services.
+ */
+public abstract class AuthCredentials {
 
-  private static class AppEngineAuthConfig extends AuthConfig {
+  private static class AppEngineAuthCredentials extends AuthCredentials {
 
     @Override
     protected HttpRequestInitializer httpRequestInitializer(
@@ -43,17 +48,17 @@ protected HttpRequestInitializer httpRequestInitializer(
     }
   }
 
-  private static class ServiceAccountAuthConfig extends AuthConfig {
+  private static class ServiceAccountAuthCredentials extends AuthCredentials {
 
     private final String account;
     private final PrivateKey privateKey;
 
-    ServiceAccountAuthConfig(String account, PrivateKey privateKey) {
+    ServiceAccountAuthCredentials(String account, PrivateKey privateKey) {
       this.account = checkNotNull(account);
       this.privateKey = checkNotNull(privateKey);
     }
 
-    ServiceAccountAuthConfig() {
+    ServiceAccountAuthCredentials() {
       account = null;
       privateKey = null;
     }
@@ -76,13 +81,14 @@ protected HttpRequestInitializer httpRequestInitializer(
   protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
       Set scopes);
 
-  public static AuthConfig createForAppEngine() {
-    return new AppEngineAuthConfig();
+  public static AuthCredentials createForAppEngine() {
+    return new AppEngineAuthCredentials();
   }
 
-  public static AuthConfig createForComputeEngine() throws IOException, GeneralSecurityException {
+  public static AuthCredentials createForComputeEngine()
+      throws IOException, GeneralSecurityException {
     final ComputeCredential cred = getComputeCredential();
-    return new AuthConfig() {
+    return new AuthCredentials() {
       @Override
       protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
           Set scopes) {
@@ -91,12 +97,34 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
     };
   }
 
-  public static AuthConfig createFor(String account, PrivateKey privateKey) {
-    return new ServiceAccountAuthConfig(account, privateKey);
+  /**
+   * Returns the Application Default Credentials.
+   *
+   * 
Returns the Application Default Credentials which are credentials that identify and
+   * authorize the whole application. This is the built-in service account if running on Google
+   * Compute Engine or the credentials file from the path in the environment variable
+   * GOOGLE_APPLICATION_CREDENTIALS.

+   *
+   * @return the credentials instance.
+   * @throws IOException if the credentials cannot be created in the current environment.
+   */
+  public static AuthCredentials createApplicationDefaults() throws IOException {
+    final GoogleCredentials credentials = GoogleCredentials.getApplicationDefault();
+    return new AuthCredentials() {
+      @Override
+      protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
+          Set scopes) {
+        return new HttpCredentialsAdapter(credentials);
+      }
+    };
+  }
+
+  public static AuthCredentials createFor(String account, PrivateKey privateKey) {
+    return new ServiceAccountAuthCredentials(account, privateKey);
   }
 
-  public static AuthConfig noCredentials() {
-    return new ServiceAccountAuthConfig();
+  public static AuthCredentials noCredentials() {
+    return new ServiceAccountAuthCredentials();
   }
 
   static ComputeCredential getComputeCredential() throws IOException, GeneralSecurityException {
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 1c51b348abc6..3e655d853e52 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -39,14 +39,14 @@ public abstract class ServiceOptions {
 
   private final String host;
   private final HttpTransport httpTransport;
-  private final AuthConfig authConfig;
+  private final AuthCredentials authCredentials;
   private final RetryParams retryParams;
 
   protected abstract static class Builder> {
 
     private String host;
     private HttpTransport httpTransport;
-    private AuthConfig authConfig;
+    private AuthCredentials authCredentials;
     private RetryParams retryParams;
 
     protected Builder() {}
@@ -54,7 +54,7 @@ protected Builder() {}
     protected Builder(ServiceOptions options) {
       host = options.host;
       httpTransport = options.httpTransport;
-      authConfig = options.authConfig;
+      authCredentials = options.authCredentials;
       retryParams = options.retryParams;
     }
 
@@ -75,8 +75,8 @@ public B httpTransport(HttpTransport httpTransport) {
       return self();
     }
 
-    public B authConfig(AuthConfig authConfig) {
-      this.authConfig = authConfig;
+    public B authConfig(AuthCredentials authCredentials) {
+      this.authCredentials = authCredentials;
       return self();
     }
 
@@ -89,7 +89,7 @@ public B retryParams(RetryParams retryParams) {
   protected ServiceOptions(Builder builder) {
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransport = firstNonNull(builder.httpTransport, defaultHttpTransport());
-    authConfig = firstNonNull(builder.authConfig, defaultAuthConfig());
+    authCredentials = firstNonNull(builder.authCredentials, defaultAuthConfig());
     retryParams = builder.retryParams;
   }
 
@@ -104,29 +104,36 @@ private static HttpTransport defaultHttpTransport() {
     }
     // Consider Compute
     try {
-      return AuthConfig.getComputeCredential().getTransport();
+      return AuthCredentials.getComputeCredential().getTransport();
     } catch (Exception e) {
       // Maybe not on GCE
     }
     return new NetHttpTransport();
   }
 
-  private static AuthConfig defaultAuthConfig() {
-    // Consider App Engine
+  public static AuthCredentials defaultAuthConfig() {
+    // Consider App Engine. This will not be needed once issue #21 is fixed.
     if (appEngineAppId() != null) {
       try {
-        return AuthConfig.createForAppEngine();
+        return AuthCredentials.createForAppEngine();
       } catch (Exception ignore) {
         // Maybe not on App Engine
       }
     }
-    // Consider Compute
+
+    try {
+      return AuthCredentials.createApplicationDefaults();
+    } catch (Exception ex) {
+      // fallback to old-style
+    }
+
+    // Consider old-style Compute. This will not be needed once issue #21 is fixed.
     try {
-      return AuthConfig.createForComputeEngine();
+      return AuthCredentials.createForComputeEngine();
     } catch (Exception ignore) {
       // Maybe not on GCE
     }
-    return AuthConfig.noCredentials();
+    return AuthCredentials.noCredentials();
   }
 
   protected static String appEngineAppId() {
@@ -176,15 +183,15 @@ public HttpTransport httpTransport() {
     return httpTransport;
   }
 
-  public AuthConfig authConfig() {
-    return authConfig;
+  public AuthCredentials authConfig() {
+    return authCredentials;
   }
 
   public RetryParams retryParams() {
     return retryParams;
   }
 
-  protected HttpRequestInitializer httpRequestInitializer() {
+  public HttpRequestInitializer httpRequestInitializer() {
     return authConfig().httpRequestInitializer(httpTransport, scopes());
   }
 
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
deleted file mode 100644
index 45b2f43efe44..000000000000
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpc.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;

/**
 * Provides access to the remote Datastore service.
 */
public interface DatastoreRpc {

  public class DatastoreRpcException extends Exception {

    /**
     * The reason for the exception.
     *
     * @see Google Cloud Datastore error codes
     */
    public enum Reason {

      ABORTED(true, "Request aborted", 409),
      DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
      FAILED_PRECONDITION(false, "Invalid request", 412),
      INTERNAL(false, "Server returned an error", 500),
      INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
      PERMISSION_DENIED(false, "Unauthorized request", 403),
      RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
      UNAVAILABLE(true, "Could not reach service", 503);

      private final boolean retryable;
      private final String description;
      private final int httpStatus;

      private Reason(boolean retryable, String description, int httpStatus) {
        this.retryable = retryable;
        this.description = description;
        this.httpStatus = httpStatus;
      }

      public boolean retryable() {
        return retryable;
      }

      public String description() {
        return description;
      }

      public int httpStatus() {
        return httpStatus;
      }
    }

    private final String reason;
    private final int httpStatus;
    private final boolean retryable;

    public DatastoreRpcException(Reason reason) {
      this(reason.name(), reason.httpStatus, reason.retryable, reason.description);
    }

    public DatastoreRpcException(String reason, int httpStatus, boolean retryable, String message) {
      super(message);
      this.reason = reason;
      this.httpStatus = httpStatus;
      this.retryable = retryable;
    }

    public String reason() {
      return reason;
    }

    public int httpStatus() {
      return httpStatus;
    }

    public boolean retryable() {
      return retryable;
    }
  }

  AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException;

  BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException;

  CommitResponse commit(CommitRequest request) throws DatastoreRpcException;

  LookupResponse lookup(LookupRequest request) throws DatastoreRpcException;

  RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException;

  RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
deleted file mode 100644
index 7bcabf081ef0..000000000000
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceOptions;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.dataset())
            .host(options.host())
            .initializer(options.httpTransport().createRequestFactory().getInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
deleted file mode 100644
index 7d119178213a..000000000000
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DefaultStorageRpc.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public Bucket bucket(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java b/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
deleted file mode 100644
index 30caacc6df4a..000000000000
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcProvider.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.common.collect.Iterables;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.storage.StorageServiceOptions;

import java.util.ServiceLoader;

public class ServiceRpcProvider {

  public static DatastoreRpc datastore(DatastoreServiceOptions options) {
    DatastoreRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(DatastoreRpcFactory.class), null);
    return factory == null ? new DefaultDatastoreRpc(options) : factory.create(options);
  }

  public static StorageRpc storage(StorageServiceOptions options) {
    StorageRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(StorageRpcFactory.class), null);
    return factory == null ? new DefaultStorageRpc(options) : factory.create(options);
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 3a407249f47b..4f41d2112dba 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -88,6 +88,7 @@ private B self() {
       return (B) this;
     }
 
+    @SuppressWarnings("unchecked")
     protected B fill(DatastoreV1.Entity entityPb) {
       Map> copiedProperties = Maps.newHashMap();
       for (DatastoreV1.Property property : entityPb.getPropertyList()) {
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index 14301a21dc86..69094456a0c0 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -20,8 +20,8 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.RetryHelper;
 import com.google.gcloud.RetryHelper.RetryHelperException;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
 
 import java.util.HashMap;
 import java.util.Map;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 119a29a38792..1b1bf81c3e2c 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -30,8 +30,8 @@
 import com.google.gcloud.RetryHelper;
 import com.google.gcloud.RetryHelper.RetryHelperException;
 import com.google.gcloud.RetryParams;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
+import com.google.gcloud.spi.DatastoreRpc;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
 import com.google.protobuf.ByteString;
 
 import java.util.Arrays;
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index f57b222b6742..bc7838df4a6b 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -26,9 +26,9 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.gcloud.ServiceOptions;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
-import com.google.gcloud.com.google.gcloud.spi.ServiceRpcProvider;
+import com.google.gcloud.spi.DatastoreRpc;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
+import com.google.gcloud.spi.ServiceRpcProvider;
 
 import java.lang.reflect.Method;
 import java.util.Iterator;
diff --git a/src/main/java/com/google/gcloud/datastore/FullEntity.java b/src/main/java/com/google/gcloud/datastore/FullEntity.java
index ac20c50136b4..d7084420665e 100644
--- a/src/main/java/com/google/gcloud/datastore/FullEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/FullEntity.java
@@ -34,7 +34,7 @@ public static class Builder extends BaseEntity.Builder<
       super(key);
     }
 
-    Builder(FullEntity entity) {
+    Builder(FullEntity entity) {
       super(entity);
     }
 
diff --git a/src/main/java/com/google/gcloud/spi/DatastoreRpc.java b/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
new file mode 100644
index 000000000000..be194c7b3848
--- /dev/null
+++ b/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;

/**
 * Provides access to the remote Datastore service.
 */
public interface DatastoreRpc {

  public class DatastoreRpcException extends Exception {

    /**
     * The reason for the exception.
     *
     * @see Google Cloud Datastore error codes
     */
    public enum Reason {

      ABORTED(true, "Request aborted", 409),
      DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
      FAILED_PRECONDITION(false, "Invalid request", 412),
      INTERNAL(false, "Server returned an error", 500),
      INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
      PERMISSION_DENIED(false, "Unauthorized request", 403),
      RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
      UNAVAILABLE(true, "Could not reach service", 503);

      private final boolean retryable;
      private final String description;
      private final int httpStatus;

      private Reason(boolean retryable, String description, int httpStatus) {
        this.retryable = retryable;
        this.description = description;
        this.httpStatus = httpStatus;
      }

      public boolean retryable() {
        return retryable;
      }

      public String description() {
        return description;
      }

      public int httpStatus() {
        return httpStatus;
      }
    }

    private final String reason;
    private final int httpStatus;
    private final boolean retryable;

    public DatastoreRpcException(Reason reason) {
      this(reason.name(), reason.httpStatus, reason.retryable, reason.description);
    }

    public DatastoreRpcException(String reason, int httpStatus, boolean retryable, String message) {
      super(message);
      this.reason = reason;
      this.httpStatus = httpStatus;
      this.retryable = retryable;
    }

    public String reason() {
      return reason;
    }

    public int httpStatus() {
      return httpStatus;
    }

    public boolean retryable() {
      return retryable;
    }
  }

  AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException;

  BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException;

  CommitResponse commit(CommitRequest request) throws DatastoreRpcException;

  LookupResponse lookup(LookupRequest request) throws DatastoreRpcException;

  RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException;

  RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java b/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
similarity index 76%
rename from src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
rename to src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
index 729144e6ca26..c25a22aae717 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/DatastoreRpcFactory.java
+++ b/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.datastore.DatastoreServiceOptions;

public interface DatastoreRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.datastore.DatastoreServiceOptions;

public interface DatastoreRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
new file mode 100644
index 000000000000..df7e309c5903
--- /dev/null
+++ b/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.dataset())
            .host(options.host())
            .initializer(options.httpRequestInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
new file mode 100644
index 000000000000..3295fa003530
--- /dev/null
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public Bucket bucket(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java b/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
similarity index 79%
rename from src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
rename to src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
index 84d31d9f339c..42c8da0f20f4 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/ServiceRpcFactory.java
+++ b/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

public interface ServiceRpcFactory {

  S create(O options);
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

public interface ServiceRpcFactory {

  S create(O options);
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java b/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
new file mode 100644
index 000000000000..a9882ee231b7
--- /dev/null
+++ b/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.common.collect.Iterables;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.storage.StorageServiceOptions;

import java.util.ServiceLoader;

public class ServiceRpcProvider {

  public static DatastoreRpc datastore(DatastoreServiceOptions options) {
    DatastoreRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(DatastoreRpcFactory.class), null);
    return factory == null ? new DefaultDatastoreRpc(options) : factory.create(options);
  }

  public static StorageRpc storage(StorageServiceOptions options) {
    StorageRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(StorageRpcFactory.class), null);
    return factory == null ? new DefaultStorageRpc(options) : factory.create(options);
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
similarity index 71%
rename from src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
rename to src/main/java/com/google/gcloud/spi/StorageRpc.java
index b6384883e350..c53574986c4b 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java b/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
similarity index 77%
rename from src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
rename to src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
index c011954b6789..e12f036a3fe3 100644
--- a/src/main/java/com/google/gcloud/com/google/gcloud/spi/StorageRpcFactory.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 4c84d6f9d67b..f3cd2cb25c43 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -19,7 +19,7 @@
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.gcloud.BaseService;
-import com.google.gcloud.com.google.gcloud.spi.StorageRpc;
+import com.google.gcloud.spi.StorageRpc;
 
 import java.io.IOException;
 
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 764c0df4fcb3..a1b61bcc00d4 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -20,8 +20,8 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import com.google.gcloud.ServiceOptions;
-import com.google.gcloud.com.google.gcloud.spi.ServiceRpcProvider;
-import com.google.gcloud.com.google.gcloud.spi.StorageRpc;
+import com.google.gcloud.spi.ServiceRpcProvider;
+import com.google.gcloud.spi.StorageRpc;
 
 import java.util.Set;
 
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
index 8b164455c001..d26cc246b3fb 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retriable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retriable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
index 4614b75c9e06..1b5c0b3a72f6 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -22,7 +22,7 @@
 import static junit.framework.TestCase.assertSame;
 import static org.junit.Assert.assertTrue;
 
-import com.google.gcloud.com.google.gcloud.spi.DatastoreRpc;
+import com.google.gcloud.spi.DatastoreRpc;
 
 import org.easymock.EasyMock;
 import org.junit.Before;

From f09cae4bb0578b98df815695b0aefd49af5db1bb Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 18 Mar 2015 16:41:06 -0700
Subject: [PATCH 137/732] minor rename and version bump

---
 pom.xml                                                |  2 +-
 src/main/java/com/google/gcloud/ServiceOptions.java    | 10 +++++-----
 .../java/com/google/gcloud/datastore/BaseEntity.java   |  2 +-
 .../gcloud/datastore/DatastoreServiceOptionsTest.java  |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/pom.xml b/pom.xml
index e144308fd875..d978fe4c1afd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   com.google.gcloud
   gcloud-java
   jar
-  0.0.2
+  0.0.3
   GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 3e655d853e52..bbc8ef935a6b 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -75,7 +75,7 @@ public B httpTransport(HttpTransport httpTransport) {
       return self();
     }
 
-    public B authConfig(AuthCredentials authCredentials) {
+    public B authCredentials(AuthCredentials authCredentials) {
       this.authCredentials = authCredentials;
       return self();
     }
@@ -89,7 +89,7 @@ public B retryParams(RetryParams retryParams) {
   protected ServiceOptions(Builder builder) {
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransport = firstNonNull(builder.httpTransport, defaultHttpTransport());
-    authCredentials = firstNonNull(builder.authCredentials, defaultAuthConfig());
+    authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials());
     retryParams = builder.retryParams;
   }
 
@@ -111,7 +111,7 @@ private static HttpTransport defaultHttpTransport() {
     return new NetHttpTransport();
   }
 
-  public static AuthCredentials defaultAuthConfig() {
+  private static AuthCredentials defaultAuthCredentials() {
     // Consider App Engine. This will not be needed once issue #21 is fixed.
     if (appEngineAppId() != null) {
       try {
@@ -183,7 +183,7 @@ public HttpTransport httpTransport() {
     return httpTransport;
   }
 
-  public AuthCredentials authConfig() {
+  public AuthCredentials authCredentials() {
     return authCredentials;
   }
 
@@ -192,7 +192,7 @@ public RetryParams retryParams() {
   }
 
   public HttpRequestInitializer httpRequestInitializer() {
-    return authConfig().httpRequestInitializer(httpTransport, scopes());
+    return authCredentials().httpRequestInitializer(httpTransport, scopes());
   }
 
   public abstract Builder toBuilder();
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 4f41d2112dba..010f290ffaca 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -127,7 +127,7 @@ public B remove(String name) {
       return self();
     }
 
-    public B set(String name, Value value) {
+    public B set(String name, Value value) {
       properties.put(name, value);
       return self();
     }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
index 1b5c0b3a72f6..c847d3504580 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -82,6 +82,6 @@ public void testToBuilder() throws Exception {
     assertEquals(original.host(), copy.host());
     assertEquals(original.force(), copy.force());
     assertEquals(original.retryParams(), copy.retryParams());
-    assertEquals(original.authConfig(), copy.authConfig());
+    assertEquals(original.authCredentials(), copy.authCredentials());
   }
 }

From c866cf39ecf68dede1c39a29bbc07503d258bd4c Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 19 Mar 2015 08:36:54 -0700
Subject: [PATCH 138/732] move sign to deploy stage

---
 .travis.yml | 4 ++--
 pom.xml     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ed1e40cca546..a0f8685b91f2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,8 +7,8 @@ before_install:
 - mvn clean
 - git clone -b travis `git config --get remote.origin.url` target/travis
 - cp target/travis/settings.xml ~/.m2/settings.xml
-install: mvn install -Dgpg.skip=true
-script: mvn verify -Dgpg.skip=true
+install: mvn install
+script: mvn verify
 branches:
   only:
   - master
diff --git a/pom.xml b/pom.xml
index d978fe4c1afd..b8a7848c5025 100644
--- a/pom.xml
+++ b/pom.xml
@@ -242,7 +242,7 @@
         
           
             sign-artifacts
-            verify
+            deploy
             
               sign
             

From 7e1817e705e400f57cfac34e12af28d8fa46888f Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 19 Mar 2015 17:34:15 -0700
Subject: [PATCH 139/732] work in progress

---
 .../com/google/gcloud/storage/Bucket.java     | 24 -----
 .../com/google/gcloud/storage/BucketImpl.java |  1 -
 .../com/google/gcloud/storage/BucketInfo.java | 96 +++++++++++++++----
 .../gcloud/storage/StorageServiceImpl.java    |  5 +-
 4 files changed, 78 insertions(+), 48 deletions(-)
 delete mode 100644 src/main/java/com/google/gcloud/storage/Bucket.java
 delete mode 100644 src/main/java/com/google/gcloud/storage/BucketImpl.java

diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
deleted file mode 100644
index 13e615e8e1b5..000000000000
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.storage;
-
-public interface Bucket {
-
-
-  BucketInfo info();
-
-}
diff --git a/src/main/java/com/google/gcloud/storage/BucketImpl.java b/src/main/java/com/google/gcloud/storage/BucketImpl.java
deleted file mode 100644
index addf35b8f8de..000000000000
--- a/src/main/java/com/google/gcloud/storage/BucketImpl.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import com.google.gcloud.storage.StorageObject.Key;

import java.util.Iterator;

public class BucketImpl implements Bucket {

  private final StorageServiceImpl storageService;
  private final com.google.api.services.storage.model.Bucket model;

  BucketImpl(StorageServiceImpl storageService, com.google.api.services.storage.model.Bucket model) {
    this.storageService = storageService;
    this.model = model;
  }

  @Override
  public String id() {
    return null;
  }

  @Override
  public String name() {
    return null;
  }

  @Override
  public Cors cors() {
    return null;
  }

  @Override
  public Acl acl() {
    return null;
  }

  @Override
  public Acl defaultObjectAcl() {
    return null;
  }

  @Override
  public void updateCors(Cors cors) {

  }

  @Override
  public void updateAcl(Acl acl) {

  }

  @Override
  public void updateDefaultObjectAcl() {

  }

  @Override
  public void delete(String... objectName) {

  }

  @Override
  public void compose(Iterable sourceObjectNames, String destObjectName) {

  }

  @Override
  public void copy(String sourceObjectName, Key destObjectKey) {

  }

  @Override
  public StorageObject get(String objectName) {
    return null;
  }

  @Override
  public Iterator get(String... objectName) {
    return null;
  }

  @Override
  public ObjectReadChannel getInputChannel(String ObjectName) {
    return null;
  }

  @Override
  public ObjectWriteChannel getOutputChannel(String ObjectName) {
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index 2bf3900002c3..d82229e4008d 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -16,50 +16,89 @@
 
 package com.google.gcloud.storage;
 
+import com.google.api.services.storage.model.Bucket;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
 public final class BucketInfo {
 
   private final String id;
   private final String name;
+  private final String etag;
   private final long createTime;
   private final long metageneration;
   private final Cors cors;
-  private final Acl acl;
-  private final Acl defaultAcl;
+  private final ImmutableList acl;
+  private final ImmutableList defaultAcl;
+  private final Location location;
+  private final StorageClass storageClass;
 
   public enum StorageClass {
     DURABLE_REDUCED_AVAILABILITY,
     STANDARD
   }
 
+  public enum Location {
+    US, EU, ASIA
+  }
+
   public final static class Builder {
 
     private final String id;
     private final String name;
-    private final long createTime;
-    private final long metageneration;
+    private StorageClass storageClass;
+    private Location location;
+    private String etag;
+    private Long createTime;
+    private Long metageneration;
     private Cors cors;
-    private Acl acl;
-    private Acl defaultAcl;
+    private Iterable acl;
+    private Iterable defaultAcl;
 
-    Builder(String id, String name, long createTime, long metageneration) {
+    Builder(String id, String name) {
       this.id = id;
       this.name = name;
+    }
+
+    public Builder storageClass(StorageClass storageClass) {
+      this.storageClass = storageClass;
+      return this;
+    }
+
+    public Builder location(Location location) {
+      this.location = location;
+      return this;
+    }
+
+    Builder etag(String etag) {
+      this.etag = etag;
+      return this;
+    }
+
+    Builder createTime(Long createTime) {
       this.createTime = createTime;
+      return this;
+    }
+
+    Builder metageneration(Long metageneration) {
       this.metageneration = metageneration;
+      return this;
     }
 
-    public Builder cors(Cors cors) {
+    Builder cors(Cors cors) {
       this.cors = cors;
       return this;
     }
 
-    public Builder acl(Acl acl) {
-      this.acl = acl;
+    public Builder acl(Iterable acl) {
+      this.acl = ImmutableList.copyOf(acl);
       return this;
     }
 
-    public Builder defaultAcl(Acl defaultAcl) {
-      this.defaultAcl = defaultAcl;
+    public Builder defaultAcl(Iterable acl) {
+      this.defaultAcl = ImmutableList.copyOf(acl);
       return this;
     }
 
@@ -71,11 +110,14 @@ public BucketInfo build() {
   private BucketInfo(Builder builder) {
     id = builder.id;
     name = builder.name;
-    createTime = builder.createTime;
-    metageneration = builder.metageneration;
+    etag = builder.etag;
+    createTime = MoreObjects.firstNonNull(builder.createTime, 0L);
+    metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L);
+    location = builder.location;
+    storageClass = builder.storageClass;
     cors = builder.cors;
-    acl = builder.acl;
-    defaultAcl = builder.defaultAcl;
+    acl = ImmutableList.copyOf(builder.acl);
+    defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
   }
 
   public String id() {
@@ -90,19 +132,31 @@ public Cors cors() {
     return cors;
   }
 
-  public Acl acl() {
+  public List acl() {
     return acl;
   }
 
-  public Acl defaultObjectAcl() {
+  public List defaultObjectAcl() {
     return defaultAcl;
   }
 
   public Builder toBuilder() {
-    return builder(id, name, createTime, metageneration).cors(cors).acl(acl).defaultAcl(defaultAcl);
+    return new Builder(id, name)
+        .createTime(createTime)
+        .etag(etag)
+        .metageneration(metageneration)
+        .cors(cors)
+        .acl(acl)
+        .defaultAcl(defaultAcl)
+        .location(location)
+        .storageClass(storageClass);
+  }
+
+  void fromPb(Bucket bucket) {
+
   }
 
-  public static Builder builder(String id, String name, long createTime, long metageneration) {
-    return new Builder(id, name, createTime, metageneration);
+  Bucket toPb() {
+    return n
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 4c84d6f9d67b..8d45573e6916 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.storage;
 
+import com.google.api.services.storage.model.Bucket;
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.gcloud.BaseService;
@@ -33,7 +34,7 @@ final class StorageServiceImpl extends BaseService implem
   }
 
   @Override
-  public Iterable listBuckets() {
+  public Iterable listBuckets() {
     try {
       return Iterables.transform(storageRpc.buckets(),
           new Function() {
@@ -47,7 +48,7 @@ public Iterable listBuckets() {
   }
 
   @Override
-  public Bucket getBucket(String bucket) {
+  public BucketInfo getBucket(String bucket) {
     try {
       return new BucketImpl(this, storageRpc.bucket(bucket));
     } catch (IOException ex) {

From c3adb6e4d216300fbe3ee2aebfcb2ec56c5657e8 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 20 Mar 2015 09:14:11 -0700
Subject: [PATCH 140/732] adding examples

---
 pom.xml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/pom.xml b/pom.xml
index b8a7848c5025..ef5b96ed3de1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,6 +166,26 @@
   
   
     
+      
+        org.codehaus.mojo
+        exec-maven-plugin
+        1.3.2
+        
+          
+            
+              java
+            
+          
+        
+        
+        
+          -1
+        
+      
       
         org.apache.maven.plugins
         maven-failsafe-plugin

From ce3109ac05f58a9cdd249527a4f4b1a3d9336097 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 20 Mar 2015 17:59:58 -0700
Subject: [PATCH 141/732] examples

---
 pom.xml                                                        | 3 +++
 src/main/java/com/google/gcloud/examples/DatastoreExample.java | 1 +
 2 files changed, 4 insertions(+)
 create mode 100644 src/main/java/com/google/gcloud/examples/DatastoreExample.java

diff --git a/pom.xml b/pom.xml
index ef5b96ed3de1..4ba5b04651c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -300,6 +300,9 @@
             
               com/google/gcloud/**/*.class
             
+            
+              com/google/gcloud/example/**/*.class
+            
           
           
           256m
diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
new file mode 100644
index 000000000000..ac15f7288738
--- /dev/null
+++ b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.examples;

import com.google.gcloud.datastore.DatastoreService;
import com.google.gcloud.datastore.DatastoreServiceFactory;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.datastore.Entity;
import com.google.gcloud.datastore.Key;
import com.google.gcloud.datastore.KeyFactory;

/**
 * An example of using the Google Cloud Datastore.
 * 

 * Steps needed for running the example:
    
 * 
  1. login using gcloud SDK - {@code gcloud auth login}.
    2. 
 * 
  2. compile using maven - {@code mvn compile}
    3. 
 * 
  3. run using maven - {@code mvn exec:java
 * -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.arguments="dataset"}
    4. 
 * 

 */
public class DatastoreExample {

  public static void main(String[] args) {
    if (args.length != 1) {
      System.err.println("DatastoreExample requires one argument for dataset");
      return;
    }
    String dataset = args[0];
    DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(dataset).build();
    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
    KeyFactory keyFactory = datastore.newKeyFactory().kind("Person");
    Key key = keyFactory.newKey("Jimmy");

    System.out.println("Trying to get the entity by its key!");
    Entity entity = datastore.get(key);

    if (entity == null) {
      System.out.println("Entity not found! Creating it!");
      entity = Entity.builder(key)
          .set("age", 30L)
          .build();
      datastore.add(entity);
    }

    System.out.println("Going to modify entity: " + entity);
    Entity.Builder builder = Entity.builder(entity);
    builder.set("f", 30);
    datastore.put(entity);

    System.out.println("Trying again to get the entity by its key!");
    key = Key.builder(dataset, "Person", "Jimmy").build();
    System.out.println("Got entity: " + entity);
  }
}
\ No newline at end of file

From 018a37edaaf96caf4ae4d43145782658a28a3e58 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Mon, 23 Mar 2015 16:44:47 -0700
Subject: [PATCH 142/732] Update .travis.yml

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index a0f8685b91f2..c46dd8ceeaea 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-nguage: java
+language: java
 jdk:
 - oraclejdk8
 - oraclejdk7

From ac882e43f7dadb2a32908ef793eb74b95c065fef Mon Sep 17 00:00:00 2001
From: ludovic Champenois 
Date: Mon, 30 Mar 2015 17:01:03 -0700
Subject: [PATCH 143/732] Update DatastoreExample.java

add local usage instructions
---
 .../gcloud/examples/DatastoreExample.java     | 75 ++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index ac15f7288738..4641777df873 100644
--- a/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -1 +1,74 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.examples;

import com.google.gcloud.datastore.DatastoreService;
import com.google.gcloud.datastore.DatastoreServiceFactory;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.datastore.Entity;
import com.google.gcloud.datastore.Key;
import com.google.gcloud.datastore.KeyFactory;

/**
 * An example of using the Google Cloud Datastore.
 * 

 * Steps needed for running the example:
    
 * 
  1. login using gcloud SDK - {@code gcloud auth login}.
    2. 
 * 
  2. compile using maven - {@code mvn compile}
    3. 
 * 
  3. run using maven - {@code mvn exec:java
 * -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.arguments="dataset"}
    4. 
 * 

 */
public class DatastoreExample {

  public static void main(String[] args) {
    if (args.length != 1) {
      System.err.println("DatastoreExample requires one argument for dataset");
      return;
    }
    String dataset = args[0];
    DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(dataset).build();
    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
    KeyFactory keyFactory = datastore.newKeyFactory().kind("Person");
    Key key = keyFactory.newKey("Jimmy");

    System.out.println("Trying to get the entity by its key!");
    Entity entity = datastore.get(key);

    if (entity == null) {
      System.out.println("Entity not found! Creating it!");
      entity = Entity.builder(key)
          .set("age", 30L)
          .build();
      datastore.add(entity);
    }

    System.out.println("Going to modify entity: " + entity);
    Entity.Builder builder = Entity.builder(entity);
    builder.set("f", 30);
    datastore.put(entity);

    System.out.println("Trying again to get the entity by its key!");
    key = Key.builder(dataset, "Person", "Jimmy").build();
    System.out.println("Got entity: " + entity);
  }
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.examples;
+
+import com.google.gcloud.datastore.DatastoreService;
+import com.google.gcloud.datastore.DatastoreServiceFactory;
+import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.Key;
+import com.google.gcloud.datastore.KeyFactory;
+
+/**
+ * An example of using the Google Cloud Datastore.
+ * 

+ * Steps needed for running the example:
    
+ * 
  1. login using gcloud SDK - {@code gcloud auth login}.
    2. 
+ * 
  2. compile using maven - {@code mvn compile}
    3. 
+ * 
  3. run using maven - {@code mvn exec:java
+ * -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.arguments="dataset"}
    4. 
+ * 

+ */
+public class DatastoreExample {
+
+  public static void main(String[] args) {
+    if (args.length != 1) {
+      System.err.println("DatastoreExample requires one argument for dataset");
+      return;
+    }
+    String dataset = args[0];
+    // If you want to access a local Datastore running via the gcd sdk, do
+    //  DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+    //    .dataset(DATASET)
+    //    .host("http://localhost:" + LocalGcdHelper.PORT)
+    //    .build();
+    DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(dataset).build();
+    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
+    KeyFactory keyFactory = datastore.newKeyFactory().kind("Person");
+    Key key = keyFactory.newKey("Jimmy");
+
+    System.out.println("Trying to get the entity by its key!");
+    Entity entity = datastore.get(key);
+
+    if (entity == null) {
+      System.out.println("Entity not found! Creating it!");
+      entity = Entity.builder(key)
+          .set("age", 30L)
+          .build();
+      datastore.add(entity);
+    }
+
+    System.out.println("Going to modify entity: " + entity);
+    Entity.Builder builder = Entity.builder(entity);
+    builder.set("f", 30);
+    datastore.put(entity);
+
+    System.out.println("Trying again to get the entity by its key!");
+    key = Key.builder(dataset, "Person", "Jimmy").build();
+    System.out.println("Got entity: " + entity);
+  }
+}

From d0f4167500e9f69b6b4e1427523c8208e4becd0d Mon Sep 17 00:00:00 2001
From: Arie 
Date: Tue, 31 Mar 2015 09:19:04 -0700
Subject: [PATCH 144/732] Update GqlQuery.java

No need to escape < > inside {@code ...}
---
 .../java/com/google/gcloud/datastore/GqlQuery.java     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index e7b9275a83f9..60c22637da56 100644
--- a/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -44,9 +44,9 @@
  *
  * 
When the type of the results is known the preferred usage would be:
  * 
{@code
- *   Query<Entity> query =
+ *   Query query =
  *       Query.gqlQueryBuilder(Query.ResultType.ENTITY, "select * from kind").build();
- *   QueryResults<Entity> results = datastore.run(query);
+ *   QueryResults results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -55,10 +55,10 @@
  *
  * 
When the type of the results is unknown you can use this approach:
  * 
{@code
- *   Query<?> query = Query.gqlQueryBuilder("select __key__ from kind").build();
- *   QueryResults<?> results = datastore.run(query);
+ *   Query query = Query.gqlQueryBuilder("select __key__ from kind").build();
+ *   QueryResults results = datastore.run(query);
  *   if (Key.class.isAssignableFrom(results.resultClass())) {
- *     QueryResults<Key> keys = (QueryResults<Key>) results;
+ *     QueryResults keys = (QueryResults) results;
  *     while (keys.hasNext()) {
  *       Key key = keys.next();
  *       ...

From 8842b4d00c48bc01e5a967c0bf8d5cdeebdb0cd9 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Tue, 31 Mar 2015 09:21:26 -0700
Subject: [PATCH 145/732] Update StructuredQuery.java

No need to escape < > inside {@code ...}
---
 .../com/google/gcloud/datastore/StructuredQuery.java   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index deab06944704..3edf153e1384 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -47,8 +47,8 @@
  *
  * 
A simple query that returns all entities for a specific kind
  * 
{@code
- *   Query<Entity> query = Query.entityQueryBuilder().kind(kind).build();
- *   QueryResults<Entity> results = datastore.run(query);
+ *   Query query = Query.entityQueryBuilder().kind(kind).build();
+ *   QueryResults results = datastore.run(query);
  *   while (results.hasNext()) {
  *     Entity entity = results.next();
  *     ...
@@ -57,15 +57,15 @@
  *
  * 
A simple key-only query of all entities for a specific kind
  * 
{@code
- *   Query<Key> keyOnlyQuery =  Query.keyQueryBuilder().kind(KIND1).build();
- *   QueryResults<Key> results = datastore.run(keyOnlyQuery);
+ *   Query keyOnlyQuery =  Query.keyQueryBuilder().kind(KIND1).build();
+ *   QueryResults results = datastore.run(keyOnlyQuery);
  *   ...
  * } 

  *
  * 
A less trivial example of a projection query that returns the first 10 results
  * of "age" and "name" properties (sorted and grouped by "age") with an age greater than 18
  * 
{@code
- *   Query<ProjectionEntity> query = Query.projectionEntityQueryBuilder()
+ *   Query; query = Query.projectionEntityQueryBuilder()
  *       .kind(kind)
  *       .projection(Projection.property("age"), Projection.first("name"))
  *       .filter(PropertyFilter.gt("age", 18))

From 2c1b38197fa310d3c5923752d1c2c0c604a38132 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Tue, 31 Mar 2015 09:21:50 -0700
Subject: [PATCH 146/732] Update StructuredQuery.java

---
 src/main/java/com/google/gcloud/datastore/StructuredQuery.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index 3edf153e1384..cb141b6218b0 100644
--- a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -65,7 +65,7 @@
  * 
A less trivial example of a projection query that returns the first 10 results
  * of "age" and "name" properties (sorted and grouped by "age") with an age greater than 18
  * 
{@code
- *   Query; query = Query.projectionEntityQueryBuilder()
+ *   Query query = Query.projectionEntityQueryBuilder()
  *       .kind(kind)
  *       .projection(Projection.property("age"), Projection.first("name"))
  *       .filter(PropertyFilter.gt("age", 18))

From 9d4d7d531d6fa6af2eccb9dfb47569be20ae99b6 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Wed, 1 Apr 2015 12:05:27 -0700
Subject: [PATCH 147/732] Update README.md

Fix issue #5 and #12
---
 README.md | 89 ++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 71 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index 2ffe0742ccc7..46c1ebb5f324 100644
--- a/README.md
+++ b/README.md
@@ -1,30 +1,58 @@
-Google Cloud for Java
-=====================
+Google Cloud Java Client
+==========================
+
+Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
 
 [![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
 [![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master)
 
-Java idiomatic client for Google Cloud Platform services. Supported APIs include:
+-  [Homepage] (https://googlecloudplatform.github.io/gcloud-java/)
+-  [API Documentation] (http://googlecloudplatform.github.io/gcloud-java/apidocs)
+-  [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/examples/package-summary.html)
 
- * Google Cloud Datastore
+This client supports the following Google Cloud Platform services:
 
+-  [Google Cloud Datastore] (https://cloud.google.com/datastore/)
+
 
-> Note: This package is a work-in-progress, and may occasionally
+> Note: This client is a work-in-progress, and may occasionally
 > make backwards-incompatible changes.
 
+Quickstart
+----------
+Add this to your pom.xml file
+```xml
+
+  com.google.gcloud
+	gcloud-java
+	LATEST
+
+```
+
+
 
-## Google Cloud Datastore
+Google Cloud Datastore
+----------------------
 
-[Google Cloud Datastore][cloud-datastore] ([docs][cloud-datastore-docs]) is a fully
-managed, schemaless database for storing non-relational data. Cloud Datastore
-automatically scales with your users and supports ACID transactions, high availability
-of reads and writes, strong consistency for reads and ancestor queries, and eventual
+Google [Cloud Datastore][cloud-datastore] is a fully managed, schemaless database for
+storing non-relational data. Cloud Datastore automatically scales with
+your users and supports ACID transactions, high availability of reads and
+writes, strong consistency for reads and ancestor queries, and eventual
 consistency for all other queries.
 
-Follow the [activation instructions][cloud-datastore-activation] to use the Google
-Cloud Datastore API with your project.
+See the [Google Cloud Datastore docs][cloud-datastore-activation] for more details on how to activate
+Cloud Datastore for your project.
+
+See the ``gcloud-java`` API [datastore documentation][datastore-api] to learn how to interact
+with the Cloud Datastore using this Client Library.
 
 ```java
 import com.google.gcloud.datastore.DatastoreService;
@@ -49,15 +77,40 @@ if (entity == null) {
 }
 ```
 
-## Contributing
+Contributing
+------------
+
+Contributions to this library are always welcome and highly encouraged.
+
+See [CONTRIBUTING] for more information on how to get started.
+
+Java Versions
+-------------
+
+Java 7+is required for using this client.
+
+Versioning
+----------
+
+This library follows [Semantic Versioning] (http://semver.org/).
+
+It is currently in major version zero (``0.y.z``), which means that anything
+may change at any time and the public API should not be considered
+stable.
+
+License
+-------
+
+Apache 2.0 - See [LICENSE] for more information.
 
-Contributions are welcome. Please, see the
-[CONTRIBUTING](https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md)
-document for details.
 
-[cloud-datastore]: https://cloud.google.com/datastore/
+[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
+[cloud-platform]: https://cloud.google.com/
+[cloud-datastore]: https://cloud.google.com/datastore/docs
 [cloud-datastore-docs]: https://cloud.google.com/datastore/docs
 [cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate
+[datastore-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/datastore/package-summary.html
 
 [cloud-pubsub]: https://cloud.google.com/pubsub/
 [cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs

From 99c7c390581c25e1186ab01945b0af4b5e897d31 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Wed, 1 Apr 2015 12:07:51 -0700
Subject: [PATCH 148/732] Update README.md

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 46c1ebb5f324..103c52bb49bd 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,7 @@ Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
 This client supports the following Google Cloud Platform services:
 
 -  [Google Cloud Datastore] (https://cloud.google.com/datastore/)
+
 
@@ -87,7 +88,7 @@ See [CONTRIBUTING] for more information on how to get started.
 Java Versions
 -------------
 
-Java 7+is required for using this client.
+Java 7 or above is required for using this client.
 
 Versioning
 ----------

From 5d3520c9e7cea10a80fe2f96c15a7e3fc9609e0e Mon Sep 17 00:00:00 2001
From: Arie 
Date: Wed, 1 Apr 2015 12:08:36 -0700
Subject: [PATCH 149/732] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 103c52bb49bd..ef63a02da3bf 100644
--- a/README.md
+++ b/README.md
@@ -27,8 +27,8 @@ Add this to your pom.xml file
 ```xml
 
   com.google.gcloud
-	gcloud-java
-	LATEST
+  gcloud-java
+  LATEST
 
 ```
 

From e8510fd55b5f1ee0d5f281ea0c4095ca09e32a63 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 1 Apr 2015 14:41:47 -0700
Subject: [PATCH 150/732] work in progress

---
 .../com/google/gcloud/storage/BucketInfo.java | 98 +++++++++++++++++--
 .../java/com/google/gcloud/storage/Cors.java  | 37 ++++---
 .../google/gcloud/storage/ListOptions.java    |  2 +-
 3 files changed, 114 insertions(+), 23 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index d82229e4008d..ea79e78d325c 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -20,9 +20,12 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 
+import java.io.Serializable;
 import java.util.List;
 
-public final class BucketInfo {
+public final class BucketInfo implements Serializable {
+
+  private static final long serialVersionUID = -3946094202176916586L;
 
   private final String id;
   private final String name;
@@ -32,12 +35,41 @@ public final class BucketInfo {
   private final Cors cors;
   private final ImmutableList acl;
   private final ImmutableList defaultAcl;
-  private final Location location;
-  private final StorageClass storageClass;
+  private final String location;
+  private final String storageClass;
+
+
+  public static final class StorageClass implements Serializable {
+
+    private static final long serialVersionUID = 374002156285326563L;
+
+    private final String value;
+
+    public enum Type {
+      DURABLE_REDUCED_AVAILABILITY,
+      STANDARD,
+      OTHER
+    }
+
+    private StorageClass(Type type) {
+      value = type.name();
+    }
 
-  public enum StorageClass {
-    DURABLE_REDUCED_AVAILABILITY,
-    STANDARD
+    private StorageClass(String value) {
+      this.value = value;
+    }
+
+    public static StorageClass standard() {
+      return new StorageClass(Type.STANDARD);
+    }
+
+    public static StorageClass durableReducedAvailability() {
+      return new StorageClass(Type.DURABLE_REDUCED_AVAILABILITY);
+    }
+
+    public static StorageClass other(String other) {
+      return new StorageClass(other);
+    }
   }
 
   public enum Location {
@@ -48,8 +80,8 @@ public final static class Builder {
 
     private final String id;
     private final String name;
-    private StorageClass storageClass;
-    private Location location;
+    private String storageClass;
+    private String location;
     private String etag;
     private Long createTime;
     private Long metageneration;
@@ -63,11 +95,19 @@ public final static class Builder {
     }
 
     public Builder storageClass(StorageClass storageClass) {
+      return storageClass(storageClass.name());
+    }
+
+    Builder storageClass(String storageClass) {
       this.storageClass = storageClass;
       return this;
     }
 
     public Builder location(Location location) {
+      return location(location.name());
+    }
+
+    Builder location(String location) {
       this.location = location;
       return this;
     }
@@ -128,6 +168,26 @@ public String name() {
     return name;
   }
 
+  public String etag() {
+    return etag;
+  }
+
+  public long createTime() {
+    return createTime;
+  }
+
+  public long metageneration() {
+    return metageneration;
+  }
+
+  public String lLocation() {
+    return location;
+  }
+
+  public String storageClass() {
+    return storageClass;
+  }
+
   public Cors cors() {
     return cors;
   }
@@ -153,10 +213,30 @@ public Builder toBuilder() {
   }
 
   void fromPb(Bucket bucket) {
+    Builder builder = new Builder(bucket.getId(), bucket.getName())
+        .createTime(bucket.getTimeCreated().getValue())
+        .etag(bucket.getEtag())
+        .metageneration(bucket.getMetageneration())
+        .location(bucket.getLocation())
+        .storageClass(bucket.getStorageClass());
+
+    cors = builder.cors;
+    acl = ImmutableList.copyOf(builder.acl);
+    defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
+
 
   }
 
   Bucket toPb() {
-    return n
+    id = builder.id;
+    name = builder.name;
+    etag = builder.etag;
+    createTime = MoreObjects.firstNonNull(builder.createTime, 0L);
+    metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L);
+    location = builder.location;
+    storageClass = builder.storageClass;
+    cors = builder.cors;
+    acl = ImmutableList.copyOf(builder.acl);
+    defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index c5a944cd6ac9..040985fd77e9 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -16,13 +16,19 @@
 
 package com.google.gcloud.storage;
 
+import com.google.api.services.storage.model.Bucket;
+import com.google.common.base.Functions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
 
+import java.io.Serializable;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
 
-public final class Cors {
+public final class Cors implements Serializable {
+
+  private static final long serialVersionUID = -8637770919343335655L;
 
   private final Integer maxAgeSeconds;
   private final ImmutableList methods;
@@ -33,7 +39,9 @@ public enum Method {
     ANY, GET, HEAD, PUT, POST, DELETE
   }
 
-  public static class Origin {
+  public static class Origin implements Serializable {
+
+    private static final long serialVersionUID = -4447958124895577993L;
 
     private final URI uri;
 
@@ -49,7 +57,6 @@ public Origin(String scheme, String host, int port) {
       } catch (URISyntaxException e) {
         throw new IllegalArgumentException(e);
       }
-
     }
 
     @Override
@@ -61,8 +68,8 @@ public String toString() {
   public static final class Builder {
 
     private Integer maxAgeSeconds;
-    private ImmutableList methods;
-    private ImmutableList origins;
+    private ImmutableList methods;
+    private ImmutableList origins;
     private ImmutableList responseHeaders;
 
     private Builder() {
@@ -73,22 +80,20 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) {
       return this;
     }
 
-    public Builder methods(List methods) {
-      this.methods = ImmutableList.copyOf(methods);
-      return this;
+    public Builder methods(Iterable methods) {
+      return methods(Iterables.transform(methods, Functions.toStringFunction()));
     }
 
-    public Builder methods(Method... methods) {
+    public Builder methods(Iterable methods) {
       this.methods = ImmutableList.copyOf(methods);
       return this;
     }
 
-    public Builder origins(List origins) {
-      this.origins = ImmutableList.copyOf(origins);
-      return this;
+    public Builder origins(Iterable origins) {
+      return origins(Iterables.transform(origins, Functions.toStringFunction()));
     }
 
-    public Builder origins(Origin... origins) {
+    public Builder origins(Iterable origins) {
       this.origins = ImmutableList.copyOf(origins);
       return this;
     }
@@ -142,4 +147,10 @@ public Builder toBuilder() {
   public static Builder builder() {
     return new Builder();
   }
+
+  void fromPb(Bucket.Cors cors) {
+    Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds());
+    cors.getMethod()
+
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java
index 9ca3d9120ff1..2584a380ae40 100644
--- a/src/main/java/com/google/gcloud/storage/ListOptions.java
+++ b/src/main/java/com/google/gcloud/storage/ListOptions.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private final boolean recurse;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;

  public static class Builder {

    private boolean recurse;
    private String prefix;
    private String cursor;
    private int maxResults;

    public Builder() {
    }

    public Builder recurse(boolean recurse) {
      this.recurse = recurse;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recurse = builder.recurse;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
  }

  public boolean recurse() {
    return recurse;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;

    public Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }
}
\ No newline at end of file

From 90ed467fd28d99d7d60d5e230afa3808ad57bd79 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 1 Apr 2015 15:48:04 -0700
Subject: [PATCH 151/732] work in progress

---
 .../com/google/gcloud/storage/BucketInfo.java | 97 +++++++++++++------
 1 file changed, 67 insertions(+), 30 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index ea79e78d325c..59ecc73433af 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -16,6 +16,8 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.api.services.storage.model.Bucket;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
@@ -35,8 +37,8 @@ public final class BucketInfo implements Serializable {
   private final Cors cors;
   private final ImmutableList acl;
   private final ImmutableList defaultAcl;
-  private final String location;
-  private final String storageClass;
+  private final Location location;
+  private final StorageClass storageClass;
 
 
   public static final class StorageClass implements Serializable {
@@ -45,43 +47,84 @@ public static final class StorageClass implements Serializable {
 
     private final String value;
 
-    public enum Type {
+    public enum Options {
       DURABLE_REDUCED_AVAILABILITY,
-      STANDARD,
-      OTHER
-    }
+      STANDARD;
 
-    private StorageClass(Type type) {
-      value = type.name();
+      private final StorageClass storageClass;
+
+      Options() {
+        storageClass = new StorageClass(name());
+      }
     }
 
     private StorageClass(String value) {
-      this.value = value;
+      this.value = checkNotNull(value);
     }
 
     public static StorageClass standard() {
-      return new StorageClass(Type.STANDARD);
+      return Options.STANDARD.storageClass;
     }
 
     public static StorageClass durableReducedAvailability() {
-      return new StorageClass(Type.DURABLE_REDUCED_AVAILABILITY);
+      return Options.DURABLE_REDUCED_AVAILABILITY.storageClass;
     }
 
     public static StorageClass other(String other) {
       return new StorageClass(other);
     }
+
+    public String value() {
+      return value;
+    }
   }
 
-  public enum Location {
-    US, EU, ASIA
+  public static final class Location implements Serializable {
+
+    private static final long serialVersionUID = 9073107666838637662L;
+    private final String value;
+
+    public enum Options {
+      US, EU, ASIA;
+
+      private final Location location;
+
+      Options() {
+        location = new Location(name());
+      }
+    }
+
+    private Location(String value) {
+      this.value = checkNotNull(value);
+    }
+
+    public static Location us() {
+      return Options.US.location;
+    }
+
+    public static Location eu() {
+      return Options.EU.location;
+    }
+
+    public static Location asia() {
+      return Options.ASIA.location;
+    }
+
+    public static Location other(String other) {
+      return new Location(other);
+    }
+
+    public String value() {
+      return value;
+    }
   }
 
   public final static class Builder {
 
     private final String id;
     private final String name;
-    private String storageClass;
-    private String location;
+    private StorageClass storageClass;
+    private Location location;
     private String etag;
     private Long createTime;
     private Long metageneration;
@@ -95,19 +138,11 @@ public final static class Builder {
     }
 
     public Builder storageClass(StorageClass storageClass) {
-      return storageClass(storageClass.name());
-    }
-
-    Builder storageClass(String storageClass) {
       this.storageClass = storageClass;
       return this;
     }
 
     public Builder location(Location location) {
-      return location(location.name());
-    }
-
-    Builder location(String location) {
       this.location = location;
       return this;
     }
@@ -180,11 +215,11 @@ public long metageneration() {
     return metageneration;
   }
 
-  public String lLocation() {
+  public Location location() {
     return location;
   }
 
-  public String storageClass() {
+  public StorageClass storageClass() {
     return storageClass;
   }
 
@@ -216,17 +251,19 @@ void fromPb(Bucket bucket) {
     Builder builder = new Builder(bucket.getId(), bucket.getName())
         .createTime(bucket.getTimeCreated().getValue())
         .etag(bucket.getEtag())
-        .metageneration(bucket.getMetageneration())
-        .location(bucket.getLocation())
-        .storageClass(bucket.getStorageClass());
+        .metageneration(bucket.getMetageneration());
+        //.location(bucket.getLocation())
+        //.storageClass(bucket.getStorageClass());
 
+    /*
     cors = builder.cors;
     acl = ImmutableList.copyOf(builder.acl);
     defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
-
+*/
 
   }
 
+  /*
   Bucket toPb() {
     id = builder.id;
     name = builder.name;
@@ -238,5 +275,5 @@ Bucket toPb() {
     cors = builder.cors;
     acl = ImmutableList.copyOf(builder.acl);
     defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
-  }
+  }*/
 }

From 83b9ce01d682f34d1e5f512c55b0f21496108cd6 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 2 Apr 2015 14:28:15 -0700
Subject: [PATCH 152/732] work in progress

---
 .../com/google/gcloud/storage/BucketInfo.java | 54 +++++++++++-----
 .../java/com/google/gcloud/storage/Cors.java  | 63 ++++++++++---------
 2 files changed, 72 insertions(+), 45 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index 59ecc73433af..e59a604371fe 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -21,6 +21,7 @@
 import com.google.api.services.storage.model.Bucket;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 
 import java.io.Serializable;
 import java.util.List;
@@ -44,34 +45,43 @@ public final class BucketInfo implements Serializable {
   public static final class StorageClass implements Serializable {
 
     private static final long serialVersionUID = 374002156285326563L;
-
+    private static final ImmutableMap STRING_TO_OPTION;
     private final String value;
 
-    public enum Options {
+    public enum Option {
       DURABLE_REDUCED_AVAILABILITY,
       STANDARD;
 
       private final StorageClass storageClass;
 
-      Options() {
+      Option() {
         storageClass = new StorageClass(name());
       }
     }
 
+    static {
+      ImmutableMap.Builder map = ImmutableMap.builder();
+      for (Option option : Option.values()) {
+        map.put(option.name(), option);
+      }
+      STRING_TO_OPTION = map.build();
+    }
+
     private StorageClass(String value) {
       this.value = checkNotNull(value);
     }
 
     public static StorageClass standard() {
-      return Options.STANDARD.storageClass;
+      return Option.STANDARD.storageClass;
     }
 
     public static StorageClass durableReducedAvailability() {
-      return Options.DURABLE_REDUCED_AVAILABILITY.storageClass;
+      return Option.DURABLE_REDUCED_AVAILABILITY.storageClass;
     }
 
-    public static StorageClass other(String other) {
-      return new StorageClass(other);
+    public static StorageClass of(String value) {
+      Option option = STRING_TO_OPTION.get(value.toUpperCase());
+      return option == null ? new StorageClass(value) : option.storageClass;
     }
 
     public String value() {
@@ -82,36 +92,46 @@ public String value() {
   public static final class Location implements Serializable {
 
     private static final long serialVersionUID = 9073107666838637662L;
+    private static final ImmutableMap STRING_TO_OPTION;
     private final String value;
 
-    public enum Options {
+    public enum Option {
       US, EU, ASIA;
 
       private final Location location;
 
-      Options() {
+      Option() {
         location = new Location(name());
       }
     }
 
+    static {
+      ImmutableMap.Builder map = ImmutableMap.builder();
+      for (Option option : Option.values()) {
+        map.put(option.name(), option);
+      }
+      STRING_TO_OPTION = map.build();
+    }
+
     private Location(String value) {
       this.value = checkNotNull(value);
     }
 
     public static Location us() {
-      return Options.US.location;
+      return Option.US.location;
     }
 
     public static Location eu() {
-      return Options.EU.location;
+      return Option.EU.location;
     }
 
     public static Location asia() {
-      return Options.ASIA.location;
+      return Option.ASIA.location;
     }
 
-    public static Location other(String other) {
-      return new Location(other);
+    public static Location of(String value) {
+      Option option = STRING_TO_OPTION.get(value.toUpperCase());
+      return option == null ? new Location(value) : option.location;
     }
 
     public String value() {
@@ -251,9 +271,9 @@ void fromPb(Bucket bucket) {
     Builder builder = new Builder(bucket.getId(), bucket.getName())
         .createTime(bucket.getTimeCreated().getValue())
         .etag(bucket.getEtag())
-        .metageneration(bucket.getMetageneration());
-        //.location(bucket.getLocation())
-        //.storageClass(bucket.getStorageClass());
+        .metageneration(bucket.getMetageneration())
+        .location(Location.of(bucket.getLocation()))
+        .storageClass(StorageClass.of(bucket.getStorageClass()));
 
     /*
     cors = builder.cors;
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index 040985fd77e9..b8d45f12c501 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -16,8 +16,12 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.api.services.storage.model.Bucket;
+import com.google.common.base.Function;
 import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 
@@ -26,6 +30,8 @@
 import java.net.URISyntaxException;
 import java.util.List;
 
+import javax.annotation.Nullable;
+
 public final class Cors implements Serializable {
 
   private static final long serialVersionUID = -8637770919343335655L;
@@ -42,34 +48,44 @@ public enum Method {
   public static class Origin implements Serializable {
 
     private static final long serialVersionUID = -4447958124895577993L;
+    private static final String ANY_URI = "*";
+    private final String value;
 
-    private final URI uri;
+    private static final Origin ANY = new Origin(ANY_URI);
 
-    public static final Origin ANY = new Origin();
+    private Origin(String value) {
+      this.value = checkNotNull(value);
+    }
 
-    private Origin() {
-      uri = null;
+    public static Origin any() {
+      return ANY;
     }
 
-    public Origin(String scheme, String host, int port) {
+    public static Origin of(String scheme, String host, int port) {
       try {
-        this.uri = new URI(scheme, null, host, port, null, null, null);
+        of(new URI(scheme, null, host, port, null, null, null).toString());
       } catch (URISyntaxException e) {
         throw new IllegalArgumentException(e);
       }
     }
 
-    @Override
-    public String toString() {
-      return uri == null ? "*" : uri.toString();
+    public static Origin of(String value) {
+      if (ANY_URI.equals(value)) {
+        return any();
+      }
+      return new Origin(value);
+    }
+
+    public String value() {
+      return value;
     }
   }
 
   public static final class Builder {
 
     private Integer maxAgeSeconds;
-    private ImmutableList methods;
-    private ImmutableList origins;
+    private ImmutableList methods;
+    private ImmutableList origins;
     private ImmutableList responseHeaders;
 
     private Builder() {
@@ -81,33 +97,20 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) {
     }
 
     public Builder methods(Iterable methods) {
-      return methods(Iterables.transform(methods, Functions.toStringFunction()));
-    }
-
-    public Builder methods(Iterable methods) {
       this.methods = ImmutableList.copyOf(methods);
       return this;
     }
 
     public Builder origins(Iterable origins) {
-      return origins(Iterables.transform(origins, Functions.toStringFunction()));
-    }
-
-    public Builder origins(Iterable origins) {
       this.origins = ImmutableList.copyOf(origins);
       return this;
     }
 
-    public Builder responseHeaders(List headers) {
+    public Builder responseHeaders(Iterable headers) {
       this.responseHeaders = ImmutableList.copyOf(headers);
       return this;
     }
 
-    public Builder responseHeaders(String... responseHeaders) {
-      this.responseHeaders = ImmutableList.copyOf(responseHeaders);
-      return this;
-    }
-
     public Cors build() {
       return new Cors(this);
     }
@@ -148,9 +151,13 @@ public static Builder builder() {
     return new Builder();
   }
 
-  void fromPb(Bucket.Cors cors) {
+  Cors fromPb(Bucket.Cors cors) {
     Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds());
-    cors.getMethod()
-
+    builder.methods(Iterables.transform(cors.getMethod(), new Function() {
+          @Override public Method apply(String name) {
+            return Method.valueOf(name);
+          }
+        }));
+    builder
   }
 }

From 1d77e082e074756a0812ed7b7e91b34269f274fd Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 2 Apr 2015 17:40:57 -0700
Subject: [PATCH 153/732] wip

---
 .../java/com/google/gcloud/storage/Acl.java   |  36 ++++++
 .../com/google/gcloud/storage/BucketInfo.java | 106 +++++++++++++-----
 .../java/com/google/gcloud/storage/Cors.java  |  42 +++++--
 3 files changed, 143 insertions(+), 41 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 15d9444311d5..5e893b828b04 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -16,6 +16,10 @@
 
 package com.google.gcloud.storage;
 
+import com.google.api.services.storage.model.BucketAccessControl;
+import com.google.api.services.storage.model.ObjectAccessControl;
+import com.google.gcloud.storage.Acl.Project.ProjectRole;
+
 import java.io.Serializable;
 
 public abstract class Acl implements Serializable {
@@ -148,4 +152,36 @@ public Type type() {
   public Role role() {
     return role;
   }
+
+  public static Acl fromPb(ObjectAccessControl objectAccessControl) {
+    Role role = Role.valueOf(objectAccessControl.getRole());
+    return forEntity(objectAccessControl.getEntity(), role);
+  }
+
+  private static Acl forEntity(String entity, Role role) {
+    if (entity.startsWith("user-")) {
+      return User.of(entity.substring(5), role);
+    }
+    if (entity.equals(User.ALL_USERS)) {
+      return User.ofAllUsers(role);
+    }
+    if (entity.equals(User.ALL_AUTHENTICATED_USERS)) {
+      return User.ofAllAuthenticatedUsers(role);
+    }
+    if (entity.startsWith("group-")) {
+      return Group.of(entity.substring(6), role);
+    }
+    if (entity.startsWith("project-")) {
+      int idx = entity.indexOf('-', 9);
+      String team = entity.substring(9, idx);
+      String projectId = entity.substring(idx + 1);
+      return Project.of(ProjectRole.valueOf(team.toUpperCase()), projectId, role);
+    }
+    return null;
+  }
+
+  public static Acl fromPb(BucketAccessControl bucketAccessControl) {
+    Role role = Role.valueOf(bucketAccessControl.getRole());
+    return forEntity(bucketAccessControl.getEntity(), role);
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index e59a604371fe..c12d54c1906a 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -17,11 +17,18 @@
 package com.google.gcloud.storage;
 
 import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Lists.newArrayList;
 
+import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.BucketAccessControl;
+import com.google.api.services.storage.model.ObjectAccessControl;
+import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
 
 import java.io.Serializable;
 import java.util.List;
@@ -35,7 +42,7 @@ public final class BucketInfo implements Serializable {
   private final String etag;
   private final long createTime;
   private final long metageneration;
-  private final Cors cors;
+  private final ImmutableList cors;
   private final ImmutableList acl;
   private final ImmutableList defaultAcl;
   private final Location location;
@@ -84,6 +91,11 @@ public static StorageClass of(String value) {
       return option == null ? new StorageClass(value) : option.storageClass;
     }
 
+    @Override
+    public String toString() {
+      return value();
+    }
+
     public String value() {
       return value;
     }
@@ -134,6 +146,11 @@ public static Location of(String value) {
       return option == null ? new Location(value) : option.location;
     }
 
+    @Override
+    public String toString() {
+      return value();
+    }
+
     public String value() {
       return value;
     }
@@ -148,9 +165,9 @@ public final static class Builder {
     private String etag;
     private Long createTime;
     private Long metageneration;
-    private Cors cors;
-    private Iterable acl;
-    private Iterable defaultAcl;
+    private Iterable cors = ImmutableList.of();
+    private Iterable acl = ImmutableList.of();
+    private Iterable defaultAcl = ImmutableList.of();
 
     Builder(String id, String name) {
       this.id = id;
@@ -182,7 +199,7 @@ Builder metageneration(Long metageneration) {
       return this;
     }
 
-    Builder cors(Cors cors) {
+    Builder cors(Iterable cors) {
       this.cors = cors;
       return this;
     }
@@ -210,7 +227,7 @@ private BucketInfo(Builder builder) {
     metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L);
     location = builder.location;
     storageClass = builder.storageClass;
-    cors = builder.cors;
+    cors = ImmutableList.copyOf(builder.cors);
     acl = ImmutableList.copyOf(builder.acl);
     defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
   }
@@ -243,7 +260,7 @@ public StorageClass storageClass() {
     return storageClass;
   }
 
-  public Cors cors() {
+  public List cors() {
     return cors;
   }
 
@@ -251,7 +268,7 @@ public List acl() {
     return acl;
   }
 
-  public List defaultObjectAcl() {
+  public List defaultAcl() {
     return defaultAcl;
   }
 
@@ -267,33 +284,64 @@ public Builder toBuilder() {
         .storageClass(storageClass);
   }
 
-  void fromPb(Bucket bucket) {
+  BucketInfo fromPb(Bucket bucket) {
     Builder builder = new Builder(bucket.getId(), bucket.getName())
         .createTime(bucket.getTimeCreated().getValue())
         .etag(bucket.getEtag())
         .metageneration(bucket.getMetageneration())
         .location(Location.of(bucket.getLocation()))
-        .storageClass(StorageClass.of(bucket.getStorageClass()));
-
-    /*
-    cors = builder.cors;
-    acl = ImmutableList.copyOf(builder.acl);
-    defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
-*/
-
+        .storageClass(StorageClass.of(bucket.getStorageClass()))
+        .cors(transform(bucket.getCors(), new Function() {
+          @Override public Cors apply(Bucket.Cors cors) {
+            return Cors.fromPb(cors);
+          }
+        }))
+        .acl(transform(bucket.getAcl(), new Function() {
+          @Override public Acl apply(BucketAccessControl bucketAccessControl) {
+            return Acl.fromPb(bucketAccessControl);
+          }
+        }))
+        .defaultAcl(transform(bucket.getDefaultObjectAcl(), new Function() {
+          @Override public Acl apply(ObjectAccessControl objectAccessControl) {
+            return Acl.fromPb(objectAccessControl);
+          }
+        }));
+    return builder.build();
   }
 
-  /*
   Bucket toPb() {
-    id = builder.id;
-    name = builder.name;
-    etag = builder.etag;
-    createTime = MoreObjects.firstNonNull(builder.createTime, 0L);
-    metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L);
-    location = builder.location;
-    storageClass = builder.storageClass;
-    cors = builder.cors;
-    acl = ImmutableList.copyOf(builder.acl);
-    defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
-  }*/
+    Bucket bucket = new Bucket();
+    bucket.setId(id);
+    bucket.setName(name);
+    bucket.setEtag(etag);
+    if (createTime > 0) {
+      bucket.setTimeCreated(new DateTime(createTime));
+    }
+    if (metageneration > 0) {
+      bucket.setMetageneration(metageneration);
+    }
+    if (location != null) {
+      bucket.setLocation(location.value());
+    }
+    if (storageClass != null) {
+      bucket.setStorageClass(storageClass.value());
+    }
+    bucket.setCors(newArrayList(Iterables.transform(cors, new Function() {
+      @Override public Bucket.Cors apply(Cors cors) {
+        return cors.toPb();
+      }
+    })));
+    bucket.setAcl(newArrayList(Iterables.transform(acl, new Function() {
+      @Override public BucketAccessControl apply(Acl acl) {
+        return new BucketAccessControl().setEntity(acl.toEntity());
+      }
+    })));
+    bucket.setDefaultObjectAcl(
+        newArrayList(Iterables.transform(defaultAcl, new Function() {
+          @Override public ObjectAccessControl apply(Acl acl) {
+            return new ObjectAccessControl().setEntity(acl.toEntity());
+          }
+        })));
+    return bucket;
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index b8d45f12c501..a58331d5063a 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -17,21 +17,19 @@
 package com.google.gcloud.storage;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Lists.newArrayList;
 
 import com.google.api.services.storage.model.Bucket;
 import com.google.common.base.Function;
 import com.google.common.base.Functions;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
 
 import java.io.Serializable;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
 
-import javax.annotation.Nullable;
-
 public final class Cors implements Serializable {
 
   private static final long serialVersionUID = -8637770919343335655L;
@@ -63,7 +61,7 @@ public static Origin any() {
 
     public static Origin of(String scheme, String host, int port) {
       try {
-        of(new URI(scheme, null, host, port, null, null, null).toString());
+        return of(new URI(scheme, null, host, port, null, null, null).toString());
       } catch (URISyntaxException e) {
         throw new IllegalArgumentException(e);
       }
@@ -76,6 +74,11 @@ public static Origin of(String value) {
       return new Origin(value);
     }
 
+    @Override
+    public String toString() {
+      return value();
+    }
+
     public String value() {
       return value;
     }
@@ -151,13 +154,28 @@ public static Builder builder() {
     return new Builder();
   }
 
-  Cors fromPb(Bucket.Cors cors) {
+  Bucket.Cors toPb() {
+    Bucket.Cors pb = new Bucket.Cors();
+    pb.setMaxAgeSeconds(maxAgeSeconds);
+    pb.setResponseHeader(responseHeaders);
+    pb.setMethod(newArrayList(transform(methods(), Functions.toStringFunction())));
+    pb.setOrigin(newArrayList(transform(origins(), Functions.toStringFunction())));
+    return pb;
+  }
+
+  static Cors fromPb(Bucket.Cors cors) {
     Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds());
-    builder.methods(Iterables.transform(cors.getMethod(), new Function() {
-          @Override public Method apply(String name) {
-            return Method.valueOf(name);
-          }
-        }));
-    builder
+    builder.methods(transform(cors.getMethod(), new Function() {
+      @Override public Method apply(String name) {
+        return Method.valueOf(name.toUpperCase());
+      }
+    }));
+    builder.origins(transform(cors.getOrigin(), new Function() {
+      @Override public Origin apply(String value) {
+        return Origin.of(value);
+      }
+    }));
+    builder.responseHeaders(cors.getResponseHeader());
+    return builder.build();
   }
 }

From 50c48aa628f8af2a13c7ef88a125053a16c6694f Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 7 Apr 2015 15:08:13 -0700
Subject: [PATCH 154/732] Improving datastore example to include all CRUD
 operations and query

---
 pom.xml                                       |   2 +-
 .../gcloud/datastore/DatastoreHelper.java     |   2 +-
 .../com/google/gcloud/datastore/DateTime.java |   8 +-
 .../google/gcloud/datastore/KeyFactory.java   |  22 ++
 .../gcloud/examples/DatastoreExample.java     | 203 +++++++++++++++---
 .../gcloud/datastore/KeyFactoryTest.java      |  38 ++++
 6 files changed, 243 insertions(+), 32 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4ba5b04651c6..0d118a63b198 100644
--- a/pom.xml
+++ b/pom.xml
@@ -79,7 +79,7 @@
     
       com.google.apis
       google-api-services-datastore-protobuf
-      v1beta2-rev1-2.1.0
+      v1beta2-rev1-2.1.2
       compile
     
     
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index 0ae0cbea218a..acc4c1065a38 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -47,7 +47,7 @@ static Entity add(DatastoreWriter writer, FullEntity entity) {
   }
 
   static KeyFactory newKeyFactory(DatastoreServiceOptions options) {
-    return new KeyFactory(options.dataset()).namespace(options.namespace());
+    return new KeyFactory(options.dataset(), options.namespace());
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/datastore/DateTime.java b/src/main/java/com/google/gcloud/datastore/DateTime.java
index 97df17ba840b..853856c1b696 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTime.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTime.java
@@ -33,7 +33,8 @@
  *
  * @see Google Cloud Datastore Entities, Properties, and Keys
  */
-public final class DateTime extends Serializable {
+public final class DateTime extends Serializable
+    implements Comparable {
 
   private static final long serialVersionUID = 7343324797621228378L;
 
@@ -53,6 +54,11 @@ public int hashCode() {
     return (int) timestampMicroseconds;
   }
 
+  @Override
+  public int compareTo(DateTime other) {
+    return toDate().compareTo(other.toDate());
+  }
+
   @Override
   public boolean equals(Object obj) {
     return obj == this || obj instanceof DateTime
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index a7ce8b1ac8db..5a68f0b8de08 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -24,8 +24,18 @@
  */
 public final class KeyFactory extends BaseKey.Builder {
 
+  private final String ds;
+  private final String ns;
+
   public KeyFactory(String dataset) {
+    this(dataset, null);
+  }
+
+  public KeyFactory(String dataset, String namespace) {
     super(dataset);
+    namespace(namespace);
+    this.ds = dataset;
+    this.ns = namespace;
   }
 
   public IncompleteKey newKey() {
@@ -46,6 +56,18 @@ public Key newKey(long id) {
     return new Key(dataset, namespace, path);
   }
 
+  /**
+   * Resets the KeyFactory to its initial state.
+   * @return {@code this} for chaining.
+   */
+  public KeyFactory reset() {
+    dataset(ds);
+    namespace(ns);
+    kind = null;
+    ancestors.clear();
+    return this;
+  }
+
   @Override
   protected IncompleteKey build() {
     return newKey();
diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index 4641777df873..6509f672df50 100644
--- a/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -19,56 +19,201 @@
 import com.google.gcloud.datastore.DatastoreService;
 import com.google.gcloud.datastore.DatastoreServiceFactory;
 import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.DateTime;
 import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.FullEntity;
+import com.google.gcloud.datastore.IncompleteKey;
 import com.google.gcloud.datastore.Key;
 import com.google.gcloud.datastore.KeyFactory;
+import com.google.gcloud.datastore.Query;
+import com.google.gcloud.datastore.Query.ResultType;
+import com.google.gcloud.datastore.QueryResults;
+import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
+import com.google.gcloud.datastore.Transaction;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
 
 /**
  * An example of using the Google Cloud Datastore.
  * 

+ * This example adds, display or clear comments for a given user.
+ * 

  * Steps needed for running the example:
    
  * 
  1. login using gcloud SDK - {@code gcloud auth login}.
    2. 
  * 
  2. compile using maven - {@code mvn compile}
    3. 
  * 
  3. run using maven - {@code mvn exec:java
- * -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample" -Dexec.arguments="dataset"}
    4. 
+ * -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample"
+ * -Dexec.args="dataset [user] [delete|display|add comment]"}
  * 

  */
 public class DatastoreExample {
 
-  public static void main(String[] args) {
-    if (args.length != 1) {
-      System.err.println("DatastoreExample requires one argument for dataset");
-      return;
+  private final static String USER_KIND = "_DS_EXAMPLE_USER";
+  private final static String COMMENT_KIND = "_DS_EXAMPLE_COMMENT";
+  private final static String NAMESPACE = "gcloud_java_example";
+  private final static String DEFAULT_ACTION = "display";
+  private final static Map ACTIONS = new HashMap<>();
+
+  private interface DatastoreAction {
+    void run(Transaction tx, Key userKey, String... args);
+    String getRequiredParams();
+  }
+
+  private static class DeleteAction implements DatastoreAction {
+    @Override
+    public void run(Transaction tx, Key userKey, String... args) {
+      Entity user = tx.get(userKey);
+      if (user == null) {
+        System.out.println("Nothing to delete, user does not exists.");
+        return;
+      }
+      Query query = Query.keyQueryBuilder()
+          .namespace(NAMESPACE)
+          .kind(COMMENT_KIND)
+          .filter(PropertyFilter.hasAncestor(userKey))
+          .build();
+      QueryResults comments = tx.run(query);
+      int count = 0;
+      while (comments.hasNext()) {
+        tx.delete(comments.next());
+        count++;
+      }
+      tx.delete(userKey);
+      System.out.printf("Deleting user '%s' and %d comments.\n", userKey.name(), count);
+    }
+
+    @Override
+    public String getRequiredParams() {
+      return "";
+    }
+  }
+
+  private static class DisplayAction implements DatastoreAction {
+    @Override
+    public void run(Transaction tx, Key userKey, String... args) {
+      Entity user = tx.get(userKey);
+      if (user == null) {
+        System.out.println("No comments for '" + userKey.name() + "'.");
+        return;
+      }
+      System.out.println("User: " + userKey.name());
+      // ORDER BY timestamp";
+      String gql = "SELECT * FROM " + COMMENT_KIND + " WHERE __key__ HAS ANCESTOR @1";
+      Query query = Query.gqlQueryBuilder(ResultType.ENTITY, gql)
+          .namespace(NAMESPACE)
+          .addBinding(userKey)
+          .build();
+      QueryResults results = tx.run(query);
+      // We could have added "ORDER BY timestamp" to the query to avoid the sorting bellow
+      // but that would require adding an ancestor index for timestamp
+      // see: https://cloud.google.com/datastore/docs/tools/indexconfig
+      Map sortedComments = new TreeMap<>();
+      while (results.hasNext()) {
+        Entity result = results.next();
+        sortedComments.put(result.getDateTime("timestamp"), result.getString("content"));
+      }
+      for (Map.Entry entry : sortedComments.entrySet()) {
+        System.out.printf("\t%s: %s\n", entry.getKey(), entry.getValue());
+      }
     }
-    String dataset = args[0];
-    // If you want to access a local Datastore running via the gcd sdk, do
-    //  DatastoreServiceOptions options = DatastoreServiceOptions.builder()
-    //    .dataset(DATASET)
-    //    .host("http://localhost:" + LocalGcdHelper.PORT)
-    //    .build();
-    DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(dataset).build();
-    DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
-    KeyFactory keyFactory = datastore.newKeyFactory().kind("Person");
-    Key key = keyFactory.newKey("Jimmy");
 
-    System.out.println("Trying to get the entity by its key!");
-    Entity entity = datastore.get(key);
+    @Override
+    public String getRequiredParams() {
+      return "";
+    }
+  }
 
-    if (entity == null) {
-      System.out.println("Entity not found! Creating it!");
-      entity = Entity.builder(key)
-          .set("age", 30L)
+  private static class AddAction implements DatastoreAction {
+    @Override
+    public void run(Transaction tx, Key userKey, String... args) {
+      Entity user = tx.get(userKey);
+      if (user == null) {
+        System.out.println("Adding a new user.");
+        user = Entity.builder(userKey)
+            .set("count", 1L)
+            .build();
+        tx.add(user);
+      }
+      String content = "No comment.";
+      if (args.length > 0) {
+        StringBuilder stBuilder = new StringBuilder();
+        for (String arg : args) {
+          stBuilder.append(arg).append(' ');
+        }
+        stBuilder.setLength(stBuilder.length() - 1);
+        content = stBuilder.toString();
+      }
+      IncompleteKey commentKey = IncompleteKey.builder(userKey, COMMENT_KIND).build();
+      FullEntity comment = FullEntity.builder(commentKey)
+          .set("content", content)
+          .set("timestamp", DateTime.now())
           .build();
-      datastore.add(entity);
+      tx.addWithDeferredIdAllocation(comment);
+      System.out.println("Adding a comment to user '" + userKey.name() + "'.");
     }
 
-    System.out.println("Going to modify entity: " + entity);
-    Entity.Builder builder = Entity.builder(entity);
-    builder.set("f", 30);
-    datastore.put(entity);
+    @Override
+    public String getRequiredParams() {
+      return "comment";
+    }
+  }
+
+  static {
+    ACTIONS.put("delete", new DeleteAction());
+    ACTIONS.put("add", new AddAction());
+    ACTIONS.put("display", new DisplayAction());
+  }
 
-    System.out.println("Trying again to get the entity by its key!");
-    key = Key.builder(dataset, "Person", "Jimmy").build();
-    System.out.println("Got entity: " + entity);
+  public static void main(String... args) {
+    DatastoreAction action = null;
+    DatastoreService datastore = null;
+    Key key = null;
+    if (args.length > 0) {
+      String dataset = args[0];
+      // If you want to access a local Datastore running via the gcd sdk, do
+      //    DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+      //        .dataset(dataset)
+      //        .namespace(NAMESPACE)
+      //        .host("http://localhost:8080")
+      //        .build();
+      DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+          .dataset(dataset)
+          .namespace(NAMESPACE)
+          .build();
+      String name = args.length > 1 ? args[1] : System.getProperty("user.name");
+      datastore = DatastoreServiceFactory.getDefault(options);
+      KeyFactory keyFactory = datastore.newKeyFactory().kind(USER_KIND);
+      key = keyFactory.newKey(name);
+      String actionName = args.length > 2 ? args[2].toLowerCase() : DEFAULT_ACTION;
+      action = ACTIONS.get(actionName);
+    }
+    if (action == null) {
+      StringBuilder actionAndParams = new StringBuilder();
+      for (Map.Entry entry : ACTIONS.entrySet()) {
+        actionAndParams.append(entry.getKey());
+        String param = entry.getValue().getRequiredParams();
+        if (param != null && !param.isEmpty()) {
+          actionAndParams.append(' ').append(param);
+        }
+        actionAndParams.append('|');
+      }
+      actionAndParams.setLength(actionAndParams.length() - 1);
+      System.out.printf("Usage: %s dataset [user] [%s]\n",
+          DatastoreExample.class.getSimpleName(), actionAndParams);
+      return;
+    }
+    args = args.length > 3 ? Arrays.copyOfRange(args, 3, args.length): new String []{};
+    Transaction tx = datastore.newTransaction();
+    try {
+      action.run(tx, key, args);
+      tx.commit();
+    } finally {
+      if (tx.active()) {
+        tx.rollback();
+      }
+    }
   }
 }
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index 943322ad4a53..aef241cd90bf 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -17,6 +17,8 @@
 package com.google.gcloud.datastore;
 
 import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNull;
+import static org.junit.Assert.assertTrue;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -34,6 +36,42 @@ public void setUp() {
     keyFactory = new KeyFactory(DATASET).kind("k");
   }
 
+  @Test
+  public void testReset() {
+    IncompleteKey key =
+        keyFactory.dataset("ds1").namespace("ns1").ancestors(PathElement.of("p", 1)).build();
+    assertEquals("k", key.kind());
+    assertEquals("ds1", key.dataset());
+    assertEquals("ns1", key.namespace());
+    assertEquals(1, key.ancestors().size());
+
+    keyFactory.reset();
+    try {
+      keyFactory.newKey(1);
+    } catch (NullPointerException ex) {
+      assertEquals("kind must not be null", ex.getMessage());
+    }
+    keyFactory.kind("k1");
+    key = keyFactory.newKey();
+    assertEquals("k1", key.kind());
+    assertEquals(DATASET, key.dataset());
+    assertNull(key.namespace());
+    assertTrue(key.ancestors().isEmpty());
+
+    keyFactory = new KeyFactory(DATASET, "ns1").kind("k");
+    key = keyFactory.newKey();
+    assertEquals(DATASET, key.dataset());
+    assertEquals("ns1", key.namespace());
+    key = keyFactory.dataset("bla1").namespace("bla2").build();
+    assertEquals("bla1", key.dataset());
+    assertEquals("bla2", key.namespace());
+    keyFactory.reset().kind("kind");
+    key = keyFactory.newKey();
+    assertEquals(DATASET, key.dataset());
+    assertEquals("ns1", key.namespace());
+    assertEquals("kind", key.kind());
+  }
+
   @Test
   public void testNewKey() throws Exception {
     Key key = keyFactory.newKey(1);

From ecbd86b31d3ca84fdb3c63f59bbad917000c213b Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 7 Apr 2015 17:20:34 -0700
Subject: [PATCH 155/732] some refactoring, fix class-loading issue and more
 example enhancments

---
 .../google/gcloud/datastore/BaseEntity.java   |   2 +-
 .../google/gcloud/datastore/BlobValue.java    |   2 +-
 .../google/gcloud/datastore/BooleanValue.java |   2 +-
 .../gcloud/datastore/DateTimeValue.java       |   2 +-
 .../google/gcloud/datastore/DoubleValue.java  |   2 +-
 .../google/gcloud/datastore/EntityValue.java  |   2 +-
 .../com/google/gcloud/datastore/KeyValue.java |   2 +-
 .../google/gcloud/datastore/ListValue.java    |   4 +-
 .../google/gcloud/datastore/LongValue.java    |   2 +-
 .../google/gcloud/datastore/NullValue.java    |   2 +-
 .../com/google/gcloud/datastore/RawValue.java |   2 +-
 .../google/gcloud/datastore/StringValue.java  |   2 +-
 .../com/google/gcloud/datastore/Value.java    | 154 +++---------------
 .../google/gcloud/datastore/ValueBuilder.java |  42 +++++
 .../gcloud/datastore/ValueMarshaller.java     |  32 ++++
 .../google/gcloud/datastore/ValueType.java    | 111 +++++++++++++
 .../gcloud/examples/DatastoreExample.java     |  31 ++--
 .../datastore/DatastoreServiceTest.java       |   2 +-
 .../gcloud/datastore/SerializationTest.java   |  31 ++--
 .../google/gcloud/datastore/ValueTest.java    |  63 ++++---
 20 files changed, 283 insertions(+), 209 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/datastore/ValueBuilder.java
 create mode 100644 src/main/java/com/google/gcloud/datastore/ValueMarshaller.java
 create mode 100644 src/main/java/com/google/gcloud/datastore/ValueType.java

diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 010f290ffaca..74417f9bc756 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -43,7 +43,7 @@
  * An entity is Google Cloud Datastore persistent data object.
  * An entity holds one or more properties, represented by a name (as {@link String})
  * and a value (as {@link com.google.gcloud.datastore.Value}), and may be associated with a
- * key. For a list of possible values see {@link com.google.gcloud.datastore.Value.Type}.
+ * key. For a list of possible values see {@link ValueType}.
  *
  * @see Google Cloud Datastore Entities, Properties, and Keys
  */
diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/src/main/java/com/google/gcloud/datastore/BlobValue.java
index 1323986eb485..fb61c0b9ad34 100644
--- a/src/main/java/com/google/gcloud/datastore/BlobValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BlobValue.java
@@ -53,7 +53,7 @@ protected void setValue(BlobValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.BLOB);
+      super(ValueType.BLOB);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
index ac62d2d3fb27..2dd98a5013ac 100644
--- a/src/main/java/com/google/gcloud/datastore/BooleanValue.java
+++ b/src/main/java/com/google/gcloud/datastore/BooleanValue.java
@@ -53,7 +53,7 @@ protected void setValue(BooleanValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.BOOLEAN);
+      super(ValueType.BOOLEAN);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
index f6c25b9d0353..7aec5c7d3c47 100644
--- a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
@@ -53,7 +53,7 @@ protected void setValue(DateTimeValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.DATE_TIME);
+      super(ValueType.DATE_TIME);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
index 8c1e7b1f18ed..d12bbe317aef 100644
--- a/src/main/java/com/google/gcloud/datastore/DoubleValue.java
+++ b/src/main/java/com/google/gcloud/datastore/DoubleValue.java
@@ -53,7 +53,7 @@ protected void setValue(DoubleValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     public Builder() {
-      super(Type.DOUBLE);
+      super(ValueType.DOUBLE);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/src/main/java/com/google/gcloud/datastore/EntityValue.java
index 44828fc35839..add50e1747b3 100644
--- a/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -54,7 +54,7 @@ protected void setValue(EntityValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.ENTITY);
+      super(ValueType.ENTITY);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/src/main/java/com/google/gcloud/datastore/KeyValue.java
index 959deb0831d4..252f48ebc92a 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyValue.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyValue.java
@@ -53,7 +53,7 @@ protected void setValue(KeyValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     public Builder() {
-      super(Type.KEY);
+      super(ValueType.KEY);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/src/main/java/com/google/gcloud/datastore/ListValue.java
index 9314297e8a3a..a13dbb17cc69 100644
--- a/src/main/java/com/google/gcloud/datastore/ListValue.java
+++ b/src/main/java/com/google/gcloud/datastore/ListValue.java
@@ -67,12 +67,12 @@ public static final class Builder extends
     private ImmutableList.Builder> listBuilder = ImmutableList.builder();
 
     private Builder() {
-      super(Type.LIST);
+      super(ValueType.LIST);
     }
 
     public Builder addValue(Value value) {
       // see datastore_v1.proto definition for list_value
-      Preconditions.checkArgument(value.type() != Type.LIST, "Cannot contain another list");
+      Preconditions.checkArgument(value.type() != ValueType.LIST, "Cannot contain another list");
       listBuilder.add(value);
       return this;
     }
diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/src/main/java/com/google/gcloud/datastore/LongValue.java
index e69d87d05e04..43d139e90249 100644
--- a/src/main/java/com/google/gcloud/datastore/LongValue.java
+++ b/src/main/java/com/google/gcloud/datastore/LongValue.java
@@ -53,7 +53,7 @@ protected void setValue(LongValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.LONG);
+      super(ValueType.LONG);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/src/main/java/com/google/gcloud/datastore/NullValue.java
index 77c813c9068b..58fed152ffd5 100644
--- a/src/main/java/com/google/gcloud/datastore/NullValue.java
+++ b/src/main/java/com/google/gcloud/datastore/NullValue.java
@@ -53,7 +53,7 @@ protected void setValue(NullValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.NULL);
+      super(ValueType.NULL);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/RawValue.java b/src/main/java/com/google/gcloud/datastore/RawValue.java
index b2e8b9785dd8..550d27804eba 100644
--- a/src/main/java/com/google/gcloud/datastore/RawValue.java
+++ b/src/main/java/com/google/gcloud/datastore/RawValue.java
@@ -51,7 +51,7 @@ protected void setValue(RawValue from, DatastoreV1.Value.Builder to) {
   static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.RAW_VALUE);
+      super(ValueType.RAW_VALUE);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/src/main/java/com/google/gcloud/datastore/StringValue.java
index 5a944da08205..95a31e714876 100644
--- a/src/main/java/com/google/gcloud/datastore/StringValue.java
+++ b/src/main/java/com/google/gcloud/datastore/StringValue.java
@@ -53,7 +53,7 @@ protected void setValue(StringValue from, DatastoreV1.Value.Builder to) {
   public static final class Builder extends Value.BaseBuilder {
 
     private Builder() {
-      super(Type.STRING);
+      super(ValueType.STRING);
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index 79266f71816c..60d1468cb90c 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -22,8 +22,6 @@
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.InvalidProtocolBufferException;
 
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
 
@@ -38,130 +36,20 @@
 public abstract class Value extends Serializable {
 
   private static final long serialVersionUID = -1899638277588872742L;
-  private static final Map DESCRIPTOR_TO_TYPE_MAP = new HashMap<>();
 
-  private final transient Type type;
+
+  private final transient ValueType valueType;
   private final transient Boolean indexed;
   private final transient Integer meaning;
   private final transient V value;
 
-  /**
-   * The type of a property.
-   *
-   * @see Google Cloud Datastore types
-   */
-  public enum Type {
-
-    /**
-     * Represents a {@code null} value.
-     */
-    NULL(NullValue.MARSHALLER),
-
-    /**
-     * Represents a {@code string} value.
-     */
-    STRING(StringValue.MARSHALLER),
-
-    /**
-     * Represents an entity value.
-     */
-    ENTITY(EntityValue.MARSHALLER),
-
-    /**
-     * Represents a {@code list} of {@link Value}s.
-     */
-    LIST(ListValue.MARSHALLER),
-
-    /**
-     * Represents a {@code key} as a value.
-     */
-    KEY(KeyValue.MARSHALLER),
-
-    /**
-     * Represents a {@code long} value.
-     */
-    LONG(LongValue.MARSHALLER),
-
-    /**
-     * Represents a {@code double} value.
-     */
-    DOUBLE(DoubleValue.MARSHALLER),
-
-    /**
-     * Represents a {@code boolean} value.
-     */
-    BOOLEAN(BooleanValue.MARSHALLER),
-
-    /**
-     * Represents a {@link DateTime} value.
-     */
-    DATE_TIME(DateTimeValue.MARSHALLER),
-
-    /**
-     * Represents a {@link Blob} value.
-     */
-    BLOB(BlobValue.MARSHALLER),
-
-    /**
-     * Represents a raw/unparsed value.
-     */
-    RAW_VALUE(RawValue.MARSHALLER);
-
-
-    @SuppressWarnings("rawtypes") private final Marshaller marshaller;
-
-    , B extends Builder> Type(Marshaller marshaller) {
-      this.marshaller = marshaller;
-      int fieldId = marshaller.getProtoFieldId();
-      if (fieldId > 0) {
-        DESCRIPTOR_TO_TYPE_MAP.put(fieldId, this);
-      }
-    }
-
-    Marshaller getMarshaller() {
-      return marshaller;
-    }
-  }
-
-  interface Builder, B extends Builder> {
-
-    Type getType();
-
-    B mergeFrom(P other);
-
-    Boolean getIndexed();
-
-    B indexed(boolean indexed);
-
-    Integer getMeaning();
-
-    @Deprecated
-    B meaning(Integer meaning);
-
-    V get();
-
-    B set(V value);
-
-    P build();
-  }
-
-  interface BuilderFactory, B extends Builder>
+  interface BuilderFactory, B extends ValueBuilder>
       extends java.io.Serializable {
     B newBuilder(V value);
   }
 
-  interface Marshaller, B extends Builder>
-      extends java.io.Serializable {
-
-    B fromProto(DatastoreV1.Value proto);
-
-    DatastoreV1.Value toProto(P value);
-
-    int getProtoFieldId();
-  }
-
-  abstract static class BaseMarshaller, B extends Builder>
-      implements Marshaller, BuilderFactory {
+  abstract static class BaseMarshaller, B extends ValueBuilder>
+      implements ValueMarshaller, BuilderFactory {
 
     private static final long serialVersionUID = 2880767488942992985L;
 
@@ -198,20 +86,20 @@ public final DatastoreV1.Value toProto(P value) {
   }
 
   abstract static class BaseBuilder, B extends BaseBuilder>
-      implements Builder {
+      implements ValueBuilder {
 
-    private final Type type;
+    private final ValueType valueType;
     private Boolean indexed;
     private Integer meaning;
     private V value;
 
-    BaseBuilder(Type type) {
-      this.type = type;
+    BaseBuilder(ValueType valueType) {
+      this.valueType = valueType;
     }
 
     @Override
-    public Type getType() {
-      return type;
+    public ValueType getValueType() {
+      return valueType;
     }
 
     @SuppressWarnings("deprecation")
@@ -265,15 +153,15 @@ private B self() {
     public abstract P build();
   }
 
-  
, B extends BaseBuilder> Value(Builder builder) {
-    type = builder.getType();
+  
, B extends BaseBuilder> Value(ValueBuilder builder) {
+    valueType = builder.getValueType();
     indexed = builder.getIndexed();
     meaning = builder.getMeaning();
     value = builder.get();
   }
 
-  public final Type type() {
-    return type;
+  public final ValueType type() {
+    return valueType;
   }
 
   public final boolean hasIndexed() {
@@ -298,11 +186,11 @@ public final V get() {
     return value;
   }
 
-  public abstract Builder toBuilder();
+  public abstract ValueBuilder toBuilder();
 
   @Override
   public int hashCode() {
-    return Objects.hash(type, indexed, meaning, value);
+    return Objects.hash(valueType, indexed, meaning, value);
   }
 
   @Override
@@ -315,7 +203,7 @@ public boolean equals(Object obj) {
       return false;
     }
     Value other = (Value) obj;
-    return Objects.equals(type, other.type)
+    return Objects.equals(valueType, other.valueType)
         && Objects.equals(indexed, other.indexed)
         && Objects.equals(meaning, other.meaning)
         && Objects.equals(value, other.value);
@@ -331,12 +219,12 @@ static Value fromPb(DatastoreV1.Value proto) {
     for (Entry entry : proto.getAllFields().entrySet()) {
       FieldDescriptor descriptor = entry.getKey();
       if (descriptor.getName().endsWith("_value")) {
-        Type type = DESCRIPTOR_TO_TYPE_MAP.get(descriptor.getNumber());
-        if (type == null) {
+        ValueType valueType = ValueType.getByDescriptorId(descriptor.getNumber());
+        if (valueType == null) {
           // unsupported type
           return RawValue.MARSHALLER.fromProto(proto).build();
         }
-        return type.getMarshaller().fromProto(proto).build();
+        return valueType.getMarshaller().fromProto(proto).build();
       }
     }
     return NullValue.MARSHALLER.fromProto(proto).build();
diff --git a/src/main/java/com/google/gcloud/datastore/ValueBuilder.java b/src/main/java/com/google/gcloud/datastore/ValueBuilder.java
new file mode 100644
index 000000000000..99a44f19366a
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/ValueBuilder.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+/**
+ * A common interface for Value builders.
+ */
+interface ValueBuilder, B extends ValueBuilder> {
+
+  ValueType getValueType();
+
+  B mergeFrom(P other);
+
+  Boolean getIndexed();
+
+  B indexed(boolean indexed);
+
+  Integer getMeaning();
+
+  @Deprecated
+  B meaning(Integer meaning);
+
+  V get();
+
+  B set(V value);
+
+  P build();
+}
diff --git a/src/main/java/com/google/gcloud/datastore/ValueMarshaller.java b/src/main/java/com/google/gcloud/datastore/ValueMarshaller.java
new file mode 100644
index 000000000000..da1a8c77fc03
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/ValueMarshaller.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import com.google.api.services.datastore.DatastoreV1;
+
+/**
+ * A common interface for Value marshallers.
+ */
+interface ValueMarshaller, B extends ValueBuilder>
+    extends java.io.Serializable {
+
+  B fromProto(DatastoreV1.Value proto);
+
+  DatastoreV1.Value toProto(P value);
+
+  int getProtoFieldId();
+}
diff --git a/src/main/java/com/google/gcloud/datastore/ValueType.java b/src/main/java/com/google/gcloud/datastore/ValueType.java
new file mode 100644
index 000000000000..5b515d6a0901
--- /dev/null
+++ b/src/main/java/com/google/gcloud/datastore/ValueType.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * The type of a Datastore property.
+ *
+ * @see Google Cloud Datastore types
+ */
+public enum ValueType {
+
+  /**
+   * Represents a {@code null} value.
+   */
+  NULL(NullValue.MARSHALLER),
+
+  /**
+   * Represents a {@code string} value.
+   */
+  STRING(StringValue.MARSHALLER),
+
+  /**
+   * Represents an entity value.
+   */
+  ENTITY(EntityValue.MARSHALLER),
+
+  /**
+   * Represents a {@code list} of {@link Value}s.
+   */
+  LIST(ListValue.MARSHALLER),
+
+  /**
+   * Represents a {@code key} as a value.
+   */
+  KEY(KeyValue.MARSHALLER),
+
+  /**
+   * Represents a {@code long} value.
+   */
+  LONG(LongValue.MARSHALLER),
+
+  /**
+   * Represents a {@code double} value.
+   */
+  DOUBLE(DoubleValue.MARSHALLER),
+
+  /**
+   * Represents a {@code boolean} value.
+   */
+  BOOLEAN(BooleanValue.MARSHALLER),
+
+  /**
+   * Represents a {@link DateTime} value.
+   */
+  DATE_TIME(DateTimeValue.MARSHALLER),
+
+  /**
+   * Represents a {@link Blob} value.
+   */
+  BLOB(BlobValue.MARSHALLER),
+
+  /**
+   * Represents a raw/unparsed value.
+   */
+  RAW_VALUE(RawValue.MARSHALLER);
+
+
+  private static final ImmutableMap DESCRIPTOR_TO_TYPE_MAP;
+
+  @SuppressWarnings("rawtypes") private final ValueMarshaller marshaller;
+
+  static {
+    ImmutableMap.Builder builder = ImmutableMap.builder();
+    for (ValueType valueType : ValueType.values()) {
+      int fieldId = valueType.getMarshaller().getProtoFieldId();
+      if (fieldId > 0) {
+        builder.put(fieldId, valueType);
+      }
+    }
+    DESCRIPTOR_TO_TYPE_MAP = builder.build();
+  }
+
+
+  , B extends ValueBuilder> ValueType(ValueMarshaller marshaller) {
+    this.marshaller = marshaller;
+  }
+
+  ValueMarshaller getMarshaller() {
+    return marshaller;
+  }
+
+  static ValueType getByDescriptorId(int descriptorId) {
+    return DESCRIPTOR_TO_TYPE_MAP.get(descriptorId);
+  }
+}
diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index 6509f672df50..7efd3af00b3b 100644
--- a/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -51,11 +51,11 @@
  */
 public class DatastoreExample {
 
-  private final static String USER_KIND = "_DS_EXAMPLE_USER";
-  private final static String COMMENT_KIND = "_DS_EXAMPLE_COMMENT";
-  private final static String NAMESPACE = "gcloud_java_example";
-  private final static String DEFAULT_ACTION = "display";
-  private final static Map ACTIONS = new HashMap<>();
+  private static final String USER_KIND = "_DS_EXAMPLE_USER";
+  private static final String COMMENT_KIND = "_DS_EXAMPLE_COMMENT";
+  private static final String NAMESPACE = "gcloud_java_example";
+  private static final String DEFAULT_ACTION = "display";
+  private static final Map ACTIONS = new HashMap<>();
 
   private interface DatastoreAction {
     void run(Transaction tx, Key userKey, String... args);
@@ -82,7 +82,7 @@ public void run(Transaction tx, Key userKey, String... args) {
         count++;
       }
       tx.delete(userKey);
-      System.out.printf("Deleting user '%s' and %d comments.\n", userKey.name(), count);
+      System.out.printf("Deleting user '%s' and %d comment[s].%n", userKey.name(), count);
     }
 
     @Override
@@ -99,7 +99,7 @@ public void run(Transaction tx, Key userKey, String... args) {
         System.out.println("No comments for '" + userKey.name() + "'.");
         return;
       }
-      System.out.println("User: " + userKey.name());
+      System.out.printf("User '%s' has %d comment[s].%n", userKey.name(), user.getLong("count"));
       // ORDER BY timestamp";
       String gql = "SELECT * FROM " + COMMENT_KIND + " WHERE __key__ HAS ANCESTOR @1";
       Query query = Query.gqlQueryBuilder(ResultType.ENTITY, gql)
@@ -116,7 +116,7 @@ public void run(Transaction tx, Key userKey, String... args) {
         sortedComments.put(result.getDateTime("timestamp"), result.getString("content"));
       }
       for (Map.Entry entry : sortedComments.entrySet()) {
-        System.out.printf("\t%s: %s\n", entry.getKey(), entry.getValue());
+        System.out.printf("\t%s: %s%n", entry.getKey(), entry.getValue());
       }
     }
 
@@ -136,6 +136,9 @@ public void run(Transaction tx, Key userKey, String... args) {
             .set("count", 1L)
             .build();
         tx.add(user);
+      } else {
+        user = Entity.builder(user).set("count", user.getLong("count") + 1L).build();
+        tx.update(user);
       }
       String content = "No comment.";
       if (args.length > 0) {
@@ -174,11 +177,11 @@ public static void main(String... args) {
     if (args.length > 0) {
       String dataset = args[0];
       // If you want to access a local Datastore running via the gcd sdk, do
-      //    DatastoreServiceOptions options = DatastoreServiceOptions.builder()
-      //        .dataset(dataset)
-      //        .namespace(NAMESPACE)
-      //        .host("http://localhost:8080")
-      //        .build();
+//          DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+//              .dataset(dataset)
+//              .namespace(NAMESPACE)
+//              .host("http://localhost:8080")
+//              .build();
       DatastoreServiceOptions options = DatastoreServiceOptions.builder()
           .dataset(dataset)
           .namespace(NAMESPACE)
@@ -201,7 +204,7 @@ public static void main(String... args) {
         actionAndParams.append('|');
       }
       actionAndParams.setLength(actionAndParams.length() - 1);
-      System.out.printf("Usage: %s dataset [user] [%s]\n",
+      System.out.printf("Usage: %s dataset [user] [%s]%n",
           DatastoreExample.class.getSimpleName(), actionAndParams);
       return;
     }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 4234dd850044..943ac5705858 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -520,7 +520,7 @@ public void testGetArray() {
     FullEntity partial2 = entity3.getEntity("partial2");
     assertEquals(PARTIAL_ENTITY2, partial1);
     assertEquals(ENTITY2, partial2);
-    assertEquals(Value.Type.BOOLEAN, entity3.getValue("bool").type());
+    assertEquals(ValueType.BOOLEAN, entity3.getValue("bool").type());
     assertEquals(6, entity3.names().size());
     assertFalse(entity3.contains("bla"));
     try {
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 3260570a9a3b..870aee8add27 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -27,7 +27,6 @@
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
-import com.google.gcloud.datastore.Value.Type;
 
 import org.junit.Test;
 
@@ -113,26 +112,26 @@ public class SerializationTest {
   private static final ProjectionEntity PROJECTION_ENTITY = ProjectionEntity.fromPb(ENTITY1.toPb());
 
   @SuppressWarnings("rawtypes")
-  private static final Multimap TYPE_TO_VALUES =
-      ImmutableMultimap.builder()
-      .put(Type.NULL, NULL_VALUE)
-      .put(Type.KEY, KEY_VALUE)
-      .put(Type.STRING, STRING_VALUE)
-      .putAll(Type.ENTITY, EMBEDDED_ENTITY_VALUE1, EMBEDDED_ENTITY_VALUE2,
+  private static final Multimap TYPE_TO_VALUES =
+      ImmutableMultimap.builder()
+      .put(ValueType.NULL, NULL_VALUE)
+      .put(ValueType.KEY, KEY_VALUE)
+      .put(ValueType.STRING, STRING_VALUE)
+      .putAll(ValueType.ENTITY, EMBEDDED_ENTITY_VALUE1, EMBEDDED_ENTITY_VALUE2,
           EMBEDDED_ENTITY_VALUE3)
-      .put(Type.LIST, LIST_VALUE)
-      .put(Type.LONG, LONG_VALUE)
-      .put(Type.DOUBLE, DOUBLE_VALUE)
-      .put(Type.BOOLEAN, BOOLEAN_VALUE)
-      .put(Type.DATE_TIME, DATE_AND_TIME_VALUE)
-      .put(Type.BLOB, BLOB_VALUE)
-      .put(Type.RAW_VALUE, RAW_VALUE)
+      .put(ValueType.LIST, LIST_VALUE)
+      .put(ValueType.LONG, LONG_VALUE)
+      .put(ValueType.DOUBLE, DOUBLE_VALUE)
+      .put(ValueType.BOOLEAN, BOOLEAN_VALUE)
+      .put(ValueType.DATE_TIME, DATE_AND_TIME_VALUE)
+      .put(ValueType.BLOB, BLOB_VALUE)
+      .put(ValueType.RAW_VALUE, RAW_VALUE)
       .build();
 
   @Test
   public void testValues() throws Exception {
-    for (Type type : Type.values()) {
-      for (Value value : TYPE_TO_VALUES.get(type)) {
+    for (ValueType valueType : ValueType.values()) {
+      for (Value value : TYPE_TO_VALUES.get(valueType)) {
         Value copy = serializeAndDeserialize(value);
         assertEquals(value, value);
         assertEquals(value, copy);
diff --git a/src/test/java/com/google/gcloud/datastore/ValueTest.java b/src/test/java/com/google/gcloud/datastore/ValueTest.java
index c9805a349dd9..51304e9f1cac 100644
--- a/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -24,7 +24,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.primitives.Primitives;
-import com.google.gcloud.datastore.Value.Type;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -43,26 +42,26 @@ public class ValueTest {
   private static final NullValue NULL_VALUE = NullValue.of();
   private static final StringValue STRING_VALUE = StringValue.of("hello");
   private static final RawValue RAW_VALUE = RawValue.of(STRING_VALUE.toPb());
-  private static final ImmutableMap TYPES = ImmutableMap.builder()
-      .put(Type.NULL, new Object[] {NullValue.class, NULL_VALUE.get()})
-      .put(Type.KEY, new Object[] {KeyValue.class, KEY})
-      .put(Type.BLOB, new Object[] {BlobValue.class, BLOB})
-      .put(Type.BOOLEAN, new Object[] {BooleanValue.class, Boolean.TRUE})
-      .put(Type.DATE_TIME, new Object[] {DateTimeValue.class, DATE_TIME})
-      .put(Type.DOUBLE, new Object[] {DoubleValue.class, 1.25D})
-      .put(Type.ENTITY, new Object[] {EntityValue.class, ENTITY})
-      .put(Type.LIST,
+  private static final ImmutableMap TYPES = ImmutableMap.builder()
+      .put(ValueType.NULL, new Object[] {NullValue.class, NULL_VALUE.get()})
+      .put(ValueType.KEY, new Object[] {KeyValue.class, KEY})
+      .put(ValueType.BLOB, new Object[] {BlobValue.class, BLOB})
+      .put(ValueType.BOOLEAN, new Object[] {BooleanValue.class, Boolean.TRUE})
+      .put(ValueType.DATE_TIME, new Object[] {DateTimeValue.class, DATE_TIME})
+      .put(ValueType.DOUBLE, new Object[] {DoubleValue.class, 1.25D})
+      .put(ValueType.ENTITY, new Object[] {EntityValue.class, ENTITY})
+      .put(ValueType.LIST,
           new Object[] {ListValue.class, ImmutableList.of(NULL_VALUE, STRING_VALUE, RAW_VALUE)})
-      .put(Type.LONG, new Object[] {LongValue.class, 123L})
-      .put(Type.RAW_VALUE, new Object[] {RawValue.class, RAW_VALUE.get()})
-      .put(Type.STRING, new Object[] {StringValue.class, STRING_VALUE.get()})
+      .put(ValueType.LONG, new Object[] {LongValue.class, 123L})
+      .put(ValueType.RAW_VALUE, new Object[] {RawValue.class, RAW_VALUE.get()})
+      .put(ValueType.STRING, new Object[] {StringValue.class, STRING_VALUE.get()})
       .build();
 
-  private ImmutableMap> typeToValue;
+  private ImmutableMap> typeToValue;
 
   private class TestBuilder extends Value.BaseBuilder, TestBuilder> {
     TestBuilder() {
-      super(Type.LIST);
+      super(ValueType.LIST);
     }
 
     @Override
@@ -79,14 +78,14 @@ public TestBuilder toBuilder() {
 
   @Before
   public void setUp() throws Exception {
-    ImmutableMap.Builder> builder = ImmutableMap.builder();
-    for (Type type : Type.values()) {
-      Object[] values = TYPES.get(type);
+    ImmutableMap.Builder> builder = ImmutableMap.builder();
+    for (ValueType valueType : ValueType.values()) {
+      Object[] values = TYPES.get(valueType);
       Class> valueClass = (Class>) values[0];
       Object value = values[1];
       if (value == null) {
         Method method = valueClass.getMethod("of");
-        builder.put(type, (Value) method.invoke(null));
+        builder.put(valueType, (Value) method.invoke(null));
       } else {
         boolean found = false;
         for (Method method : valueClass.getDeclaredMethods()) {
@@ -96,7 +95,7 @@ public void setUp() throws Exception {
               paramType = Primitives.wrap(paramType);
             }
             if (paramType.isAssignableFrom(value.getClass())) {
-              builder.put(type, (Value) method.invoke(null, value));
+              builder.put(valueType, (Value) method.invoke(null, value));
               found = true;
               break;
             }
@@ -110,17 +109,17 @@ public void setUp() throws Exception {
 
   @Test
   public void testType() throws Exception {
-    for (Map.Entry> entry : typeToValue.entrySet()) {
+    for (Map.Entry> entry : typeToValue.entrySet()) {
       assertEquals(entry.getKey(), entry.getValue().type());
     }
   }
 
   @Test
   public void testHasIndexed() throws Exception {
-    for (Map.Entry> entry : typeToValue.entrySet()) {
-      Type type = entry.getKey();
+    for (Map.Entry> entry : typeToValue.entrySet()) {
+      ValueType valueType = entry.getKey();
       Boolean indexed = entry.getValue().hasIndexed();
-      switch (type) {
+      switch (valueType) {
         case ENTITY:
           assertTrue(indexed);
           break;
@@ -138,10 +137,10 @@ public void testHasIndexed() throws Exception {
 
   @Test
   public void testIndexed() throws Exception {
-    for (Map.Entry> entry : typeToValue.entrySet()) {
-      Type type = entry.getKey();
+    for (Map.Entry> entry : typeToValue.entrySet()) {
+      ValueType valueType = entry.getKey();
       Boolean indexed = entry.getValue().indexed();
-      switch (type) {
+      switch (valueType) {
         case ENTITY:
           assertFalse(indexed);
           break;
@@ -181,10 +180,10 @@ public void testMeaning() throws Exception {
 
   @Test
   public void testGet() throws Exception {
-    for (Map.Entry> entry : typeToValue.entrySet()) {
-      Type type = entry.getKey();
+    for (Map.Entry> entry : typeToValue.entrySet()) {
+      ValueType valueType = entry.getKey();
       Value value = entry.getValue();
-      assertEquals(TYPES.get(type)[1], value.get());
+      assertEquals(TYPES.get(valueType)[1], value.get());
     }
 
     TestBuilder builder = new TestBuilder();
@@ -196,14 +195,14 @@ public void testGet() throws Exception {
   @Test
   public void testToBuilder() throws Exception {
     Set content = Collections.singleton("bla");
-    Value.Builder builder = new TestBuilder();
+    ValueBuilder builder = new TestBuilder();
     builder.meaning(1).set(content).indexed(true);
     Value value = builder.build();
     builder = value.toBuilder();
     assertEquals(Integer.valueOf(1), value.meaning());
     assertTrue(value.hasIndexed());
     assertTrue(value.indexed());
-    assertEquals(Type.LIST, value.type());
+    assertEquals(ValueType.LIST, value.type());
     assertEquals(content, value.get());
     assertEquals(value, builder.build());
   }

From a21242fff977490eb63d3a8c454b048d30f511c4 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 7 Apr 2015 17:31:17 -0700
Subject: [PATCH 156/732] pushing 0.0.4

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 0d118a63b198..3b3a0861a4e8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
   com.google.gcloud
   gcloud-java
   jar
-  0.0.3
+  0.0.4
   GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   

From 9447eb620c078215f8cd9a660a12c9f04cd59c31 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Wed, 8 Apr 2015 09:34:25 -0700
Subject: [PATCH 157/732] Update README.md

---
 README.md | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index ef63a02da3bf..7bfa5acdf9a7 100644
--- a/README.md
+++ b/README.md
@@ -59,22 +59,29 @@ with the Cloud Datastore using this Client Library.
 import com.google.gcloud.datastore.DatastoreService;
 import com.google.gcloud.datastore.DatastoreServiceFactory;
 import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.DateTime;
 import com.google.gcloud.datastore.Entity;
 import com.google.gcloud.datastore.Key;
 import com.google.gcloud.datastore.KeyFactory;
 
-DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset("...").build();
+DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
 DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
-KeyFactory keyFactory = new KeyFactory(datastore).kind("...");
+KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
 Key key = keyFactory.newKey(keyName);
 Entity entity = datastore.get(key);
 if (entity == null) {
   entity = Entity.builder(key)
       .set("name", "John Do")
       .set("age", 30)
-      .set("updated", false)
+      .set("access_time", DateTime.now())
       .build();
   datastore.put(entity);
+} else {
+  System.out.println("Updating access_time for " + entity.getString("name"));
+  entity = Entity.builder(entity)
+      .set("access_time", DateTime.now())
+      .build();
+  datastore.update(entity);
 }
 ```
 

From be37555c3d060721cadd3a27b47a7968c25fce59 Mon Sep 17 00:00:00 2001
From: Arie 
Date: Wed, 8 Apr 2015 09:59:40 -0700
Subject: [PATCH 158/732] Update pom.xml

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 3b3a0861a4e8..06b070ecf874 100644
--- a/pom.xml
+++ b/pom.xml
@@ -301,7 +301,7 @@
               com/google/gcloud/**/*.class
             
             
-              com/google/gcloud/example/**/*.class
+              com/google/gcloud/examples/**/*.class
             
           
           

From 22fc5693ee14027489c4c5780cb2837c682832e9 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 9 Apr 2015 10:08:45 -0700
Subject: [PATCH 159/732] work in progress

---
 .../java/com/google/gcloud/storage/Acl.java   | 52 ++++++++++++++++---
 .../com/google/gcloud/storage/BucketInfo.java |  4 +-
 .../google/gcloud/storage/ListOptions.java    |  2 +-
 .../com/google/gcloud/storage/ObjectInfo.java |  5 +-
 4 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 5e893b828b04..e4cb31995c03 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -56,6 +56,10 @@ public String domain() {
       return domain;
     }
 
+    String entity() {
+      return "domain-" + domain;
+    }
+
     public static Domain of(String domain, Role role) {
       return new Domain(domain, role);
     }
@@ -75,6 +79,10 @@ public String email() {
       return email;
     }
 
+    String entity() {
+      return "group-" + email;
+    }
+
     public static Group of(String email, Role role) {
       return new Group(email, role);
     }
@@ -97,6 +105,18 @@ public String email() {
       return email;
     }
 
+    String entity() {
+      switch (email) {
+        case ALL_AUTHENTICATED_USERS:
+          return ALL_AUTHENTICATED_USERS;
+        case ALL_USERS:
+          return ALL_USERS;
+        default:
+          return "user-" + email;
+      }
+    }
+
+
     public static User of(String email, Role role) {
       return new User(email, role);
     }
@@ -135,6 +155,10 @@ public String projectId() {
       return projectId;
     }
 
+    String entity() {
+      return "project-" + pRole.name().toLowerCase() + "-" + projectId;
+    }
+
     public static Project of(ProjectRole pRole, String projectId, Role role) {
       return new Project(pRole, projectId, role);
     }
@@ -153,11 +177,32 @@ public Role role() {
     return role;
   }
 
-  public static Acl fromPb(ObjectAccessControl objectAccessControl) {
+  BucketAccessControl toBucketPb() {
+    BucketAccessControl bucketPb = new BucketAccessControl();
+    bucketPb.setRole(role().toString());
+    bucketPb.setEntity(entity());
+    return bucketPb;
+  }
+
+  ObjectAccessControl toObjectPb() {
+    ObjectAccessControl objectPb = new ObjectAccessControl();
+    objectPb.setRole(role().toString());
+    objectPb.setEntity(entity());
+    return objectPb;
+  }
+
+  abstract String entity();
+
+  static Acl fromPb(ObjectAccessControl objectAccessControl) {
     Role role = Role.valueOf(objectAccessControl.getRole());
     return forEntity(objectAccessControl.getEntity(), role);
   }
 
+  static Acl fromPb(BucketAccessControl bucketAccessControl) {
+    Role role = Role.valueOf(bucketAccessControl.getRole());
+    return forEntity(bucketAccessControl.getEntity(), role);
+  }
+
   private static Acl forEntity(String entity, Role role) {
     if (entity.startsWith("user-")) {
       return User.of(entity.substring(5), role);
@@ -179,9 +224,4 @@ private static Acl forEntity(String entity, Role role) {
     }
     return null;
   }
-
-  public static Acl fromPb(BucketAccessControl bucketAccessControl) {
-    Role role = Role.valueOf(bucketAccessControl.getRole());
-    return forEntity(bucketAccessControl.getEntity(), role);
-  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index c12d54c1906a..778d53330496 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -333,13 +333,13 @@ Bucket toPb() {
     })));
     bucket.setAcl(newArrayList(Iterables.transform(acl, new Function() {
       @Override public BucketAccessControl apply(Acl acl) {
-        return new BucketAccessControl().setEntity(acl.toEntity());
+        return acl.toBucketPb();
       }
     })));
     bucket.setDefaultObjectAcl(
         newArrayList(Iterables.transform(defaultAcl, new Function() {
           @Override public ObjectAccessControl apply(Acl acl) {
-            return new ObjectAccessControl().setEntity(acl.toEntity());
+            return acl.toObjectPb();
           }
         })));
     return bucket;
diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java
index 2584a380ae40..b1b5d96f27da 100644
--- a/src/main/java/com/google/gcloud/storage/ListOptions.java
+++ b/src/main/java/com/google/gcloud/storage/ListOptions.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;

    public Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;

    public Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
index 97183bc720ea..47134d22ad21 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java
+++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
@@ -21,10 +21,13 @@
 import com.google.common.hash.HashCode;
 import com.google.common.hash.Hashing;
 
+import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
 
-public class ObjectInfo {
+public class ObjectInfo implements Serializable {
+
+  private static final long serialVersionUID = 2228487739943277159L;
 
   private final String bucket;
   private final String name;

From a2028c090a043d7f360bb03a5689eba6bc5a404c Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 9 Apr 2015 15:27:58 -0700
Subject: [PATCH 160/732] work in progress

---
 .../gcloud/examples/StorageExample.java       | 123 ++++++++++++++++++
 .../com/google/gcloud/storage/BucketInfo.java |  19 +++
 .../com/google/gcloud/storage/ObjectInfo.java | 107 ++++++++++++---
 .../google/gcloud/storage/StorageService.java |  23 +---
 4 files changed, 232 insertions(+), 40 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/examples/StorageExample.java

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
new file mode 100644
index 000000000000..87bf24d28772
--- /dev/null
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.examples;
+
+import com.google.api.services.storage.Storage;
+import com.google.gcloud.storage.BucketInfo;
+import com.google.gcloud.storage.StorageService;
+import com.google.gcloud.storage.StorageServiceFactory;
+import com.google.gcloud.storage.StorageServiceOptions;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An example of using the Google Cloud Storage.
+ * 

+ * This example demonstrates a simple/typical usage.
+ * 

+ * Steps needed for running the example:
    
+ * 
  1. login using gcloud SDK - {@code gcloud auth login}.
    2. 
+ * 
  2. compile using maven - {@code mvn compile}
    3. 
+ * 
  3. run using maven - {@code mvn exec:java
+ * -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
+ * -Dexec.args=" [] [list|get |put  []|delete ...]"}
    4. 
+ * 

+ */
+public class StorageExample {
+
+  private static final String DEFAULT_FOLDER = "_STORAGE_EXAMPLE";
+  private static final String DEFAULT_ACTION = "list";
+  private static final Map ACTIONS = new HashMap<>();
+
+  private static abstract class StorageAction {
+
+    abstract void run(StorageService storage, BucketInfo bucket, String folder, String... args);
+
+    protected String getRequiredParams() {
+      return "";
+    }
+
+    protected String fullPath(String folder, String file) {
+      StringBuilder stringBuilder = new StringBuilder(folder);
+      if (!folder.endsWith("/")) {
+        stringBuilder.append('/');
+      }
+      return stringBuilder.append(file).toString();
+    }
+  }
+
+
+  private static class DeleteAction extends StorageAction {
+    @Override
+    public void run(StorageService storage, BucketInfo bucket, String folder, String... args) {
+      for (String file : args) {
+        storage.delete(bucket.name(), fullPath(folder, file));
+      }
+    }
+
+    @Override
+    public String getRequiredParams() {
+      return "...";
+    }
+  }
+
+  static {
+    ACTIONS.put("delete", new DeleteAction());
+  }
+
+  public static void main(String... args) {
+    StorageAction action = null;
+    StorageService storage = null;
+    BucketInfo bucketInfo = null;
+    String folder = DEFAULT_FOLDER;
+    if (args.length > 0) {
+      String bucket = args[0];
+      String actionName = DEFAULT_ACTION;
+      if (args.length > 1) {
+        folder = args[1];
+        if (args.length > 2) {
+          actionName = args[2].toLowerCase();
+        }
+        storage = StorageServiceFactory.getDefault(StorageServiceOptions.builder().build());
+        bucketInfo = storage.get(bucket);
+      }
+      action = ACTIONS.get(actionName);
+    }
+
+    if (action == null) {
+      StringBuilder actionAndParams = new StringBuilder();
+      for (Map.Entry entry : ACTIONS.entrySet()) {
+        actionAndParams.append(entry.getKey());
+        String param = entry.getValue().getRequiredParams();
+        if (param != null && !param.isEmpty()) {
+          actionAndParams.append(' ').append(param);
+        }
+        actionAndParams.append('|');
+      }
+      actionAndParams.setLength(actionAndParams.length() - 1);
+      System.out.printf("Usage: %s bucket [] [%s]%n",
+          StorageExample.class.getSimpleName(), actionAndParams);
+      return;
+    }
+
+    args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{};
+    action.run(bucketInfo, folder, args);
+    }
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index 778d53330496..7f6de0300180 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -22,6 +22,7 @@
 
 import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.Bucket.Lifecycle;
 import com.google.api.services.storage.model.BucketAccessControl;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.common.base.Function;
@@ -39,6 +40,9 @@ public final class BucketInfo implements Serializable {
 
   private final String id;
   private final String name;
+  private final String owner;
+  private final String selfLink;
+  private final boolean versioningEnabled;
   private final String etag;
   private final long createTime;
   private final long metageneration;
@@ -49,6 +53,15 @@ public final class BucketInfo implements Serializable {
   private final StorageClass storageClass;
 
 
+  public static abstract class AutoDeleteRule implements Serializable {
+
+    public enum Option {
+      AGE, CREATE_BEFORE, VERSION_LIMIT, NOT_LIVE
+    }
+
+
+  }
+
   public static final class StorageClass implements Serializable {
 
     private static final long serialVersionUID = 374002156285326563L;
@@ -306,6 +319,12 @@ BucketInfo fromPb(Bucket bucket) {
             return Acl.fromPb(objectAccessControl);
           }
         }));
+    bucket.setOwner();
+    bucket.setSelfLink();
+    bucket.setWebsite();
+    bucket.setVersioning();
+    bucket.setLifecycle(new Lifecycle.Rule().
+
     return builder.build();
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
index 47134d22ad21..05a0c901284a 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java
+++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
@@ -16,12 +16,16 @@
 
 package com.google.gcloud.storage;
 
+import com.google.api.client.util.DateTime;
+import com.google.api.services.storage.model.ObjectAccessControl;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.hash.HashCode;
-import com.google.common.hash.Hashing;
+import com.google.common.collect.Lists;
 
 import java.io.Serializable;
+import java.math.BigInteger;
 import java.util.List;
 import java.util.Map;
 
@@ -37,8 +41,8 @@ public class ObjectInfo implements Serializable {
   private final String owner;
   private final long size;
   private final String contentEncoding;
-  private final HashCode md5;
-  private final long crc32;
+  private final String md5;
+  private final String crc32c;
   private final String mediaLink;
   private final ImmutableMap metadata;
   private final long generation;
@@ -46,7 +50,6 @@ public class ObjectInfo implements Serializable {
   private final long deleteTime;
   private final long updateTime;
 
-
   public static final class Builder {
 
     private String bucket;
@@ -57,8 +60,8 @@ public static final class Builder {
     private String owner;
     private long size;
     private String contentEncoding;
-    private HashCode md5;
-    private long crc32;
+    private String md5;
+    private String crc32c;
     private String mediaLink;
     private ImmutableMap metadata;
     private long generation;
@@ -109,18 +112,13 @@ public Builder contentEncoding(String contentEncoding) {
       return this;
     }
 
-    Builder md5(HashCode md5) {
+    Builder md5(String md5) {
       this.md5 = md5;
       return this;
     }
 
-    public Builder md5(byte[] bytes) {
-      this.md5 = bytes == null ? null : Hashing.md5().hashBytes(bytes);
-      return this;
-    }
-
-    public Builder crc32(long crc32) {
-      this.crc32 = crc32;
+    public Builder crc32c(String crc32c) {
+      this.crc32c = crc32c;
       return this;
     }
 
@@ -169,7 +167,7 @@ private ObjectInfo(Builder builder) {
     size = builder.size;
     contentEncoding = builder.contentEncoding;
     md5 = builder.md5;
-    crc32 = builder.crc32;
+    crc32c = builder.crc32c;
     mediaLink = builder.mediaLink;
     metadata = builder.metadata;
     generation = builder.generation;
@@ -210,12 +208,12 @@ public String contentEncoding() {
     return contentEncoding;
   }
 
-  public byte[] md5() {
-    return md5 == null ? null : md5.asBytes();
+  public String md5() {
+    return md5;
   }
 
-  public long crc32() {
-    return crc32;
+  public String crc32c() {
+    return crc32c;
   }
 
   public String mediaLink() {
@@ -248,7 +246,7 @@ public Builder toBuilder() {
         .bucket(bucket)
         .cacheControl(cacheControl)
         .contentEncoding(contentEncoding)
-        .crc32(crc32)
+        .crc32c(crc32c)
         .contentType(contentType)
         .deleteTime(deleteTime)
         .generation(generation)
@@ -266,4 +264,71 @@ public Builder toBuilder() {
   public static Builder builder() {
     return new Builder();
   }
+
+  StorageObject toPb() {
+    StorageObject storageObject = new StorageObject();
+    storageObject.setAcl(Lists.transform(acl,
+        new Function() {
+          @Override
+          public ObjectAccessControl apply(Acl acl) {
+            return acl.toObjectPb();
+          }
+        }));
+    storageObject.setBucket(bucket);
+    storageObject.setCacheControl(cacheControl);
+    storageObject.setContentEncoding(contentEncoding);
+    storageObject.setCrc32c(crc32c);
+    storageObject.setContentType(contentType);
+    storageObject.setTimeDeleted(new DateTime(deleteTime));
+    storageObject.setGeneration(generation);
+    storageObject.setMd5Hash(md5);
+    storageObject.setMediaLink(mediaLink);
+    storageObject.setMetadata(metadata);
+    storageObject.setMetageneration(metageneration);
+    storageObject.setName(name);
+    storageObject.setOwner(owner);
+    storageObject.setUpdated(new DateTime(updateTime));
+    storageObject.setSize(BigInteger.valueOf(size));
+    storageObject.setContentDisposition();
+    storageObject.setComponentCount();
+    storageObject.setContentLanguage();
+    storageObject.setEtag();
+    storageObject.setId();
+    storageObject.setSelfLink();
+    storageObject.setStorageClass();
+    return storageObject;
+  }
+
+  static ObjectInfo fromPb(StorageObject storageObject) {
+    return builder()
+        .acl(Lists.transform(storageObject.getAcl(),
+            new Function() {
+              @Override public Acl apply(ObjectAccessControl objectAccessControl) {
+                return Acl.fromPb(objectAccessControl);
+              }
+            }))
+        .bucket(storageObject.getBucket())
+        .cacheControl(storageObject.getCacheControl())
+        .contentEncoding(storageObject.getContentEncoding())
+        .crc32c(storageObject.getCrc32c())
+        .contentType(storageObject.getContentType())
+        .deleteTime(storageObject.getTimeDeleted().getValue())
+        .generation(storageObject.getGeneration())
+        .md5(storageObject.getMd5Hash())
+        .mediaLink(storageObject.getMediaLink())
+        .metadata(storageObject.getMetadata())
+        .metageneration(storageObject.getMetageneration())
+        .name(storageObject.getName())
+        .owner(storageObject.getName())
+        .updateTime(storageObject.getUpdated().getValue())
+        .size(storageObject.getSize().longValue())
+        .build();
+    storageObject.setContentDisposition();
+    storageObject.setComponentCount();
+    storageObject.setContentLanguage();
+    storageObject.setEtag();
+    storageObject.setId();
+    storageObject.setSelfLink();
+    storageObject.setStorageClass();
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 48947537653b..f88f68cb331f 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -18,40 +18,25 @@
 
 import com.google.gcloud.Service;
 
-import java.util.Iterator;
-
 public interface StorageService extends Service {
 
-  Iterable list();
-
-  Iterable list(ListOptions settings);
-
   BucketInfo get(String bucket);
 
-  Iterator get(String... bucket);
-
   ObjectInfo get(String bucket, String object);
 
-  Iterator get(String bucket, String... object);
+  Iterable list(ListOptions settings);
 
   void update(BucketInfo bucketInfo);
 
   void update(ObjectInfo objectInfo);
 
-  void delete(String bucket, String... object);
+  void delete(String bucket, String object);
 
   void compose(String bucket, Iterable sourceObjects, String destObject);
 
   void copy(String fromBucket, String fromObject, String destBucket, String destObject);
 
+  ObjectReadChannel newReader(String bucket, String ObjectName);
 
-  ObjectReadChannel getInputChannel(String bucket, String ObjectName);
-
-  ObjectWriteChannel write(String ObjectName);
-
-  // TODO: add listing
-
-  ObjectReadChannel getInputChannel();
-
-  ObjectWriteChannel getOutputChannel();
+  ObjectWriteChannel newWriter(String ObjectName);
 }

From 403c21369ccc22a18cf7aac2968e473b57326bf5 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 9 Apr 2015 18:18:11 -0700
Subject: [PATCH 161/732] work in progress

---
 .../java/com/google/gcloud/storage/Acl.java   | 170 +++++++++---------
 .../com/google/gcloud/storage/BucketInfo.java |   2 +-
 2 files changed, 81 insertions(+), 91 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index e4cb31995c03..d2e2be0848f1 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -18,15 +18,14 @@
 
 import com.google.api.services.storage.model.BucketAccessControl;
 import com.google.api.services.storage.model.ObjectAccessControl;
-import com.google.gcloud.storage.Acl.Project.ProjectRole;
 
 import java.io.Serializable;
 
-public abstract class Acl implements Serializable {
+public final class Acl implements Serializable {
 
   private static final long serialVersionUID = 6435575339887912222L;
 
-  private final Type type;
+  private final Entity entity;
   private final Role role;
 
   public enum Role {
@@ -35,105 +34,128 @@ public enum Role {
     WRITER
   }
 
-  public enum Type {
+  public enum EntityType {
     DOMAIN,
     GROUP,
     USER,
     PROJECT
   }
 
-  public static class Domain extends Acl {
+  public static abstract class Entity implements Serializable {
 
-    private static final long serialVersionUID = -3033025857280447253L;
-    private final String domain;
+    private static final long serialVersionUID = -2707407252771255840L;
+    private final EntityType type;
+    private final String value;
 
-    Domain(String domain, Role role) {
-      super(Type.DOMAIN, role);
-      this.domain = domain;
+    Entity(EntityType type, String value) {
+      this.type = type;
+      this.value = value;
     }
 
-    public String domain() {
-      return domain;
+    public EntityType type() {
+      return type;
     }
 
-    String entity() {
-      return "domain-" + domain;
+    protected String value() {
+      return value;
     }
 
-    public static Domain of(String domain, Role role) {
-      return new Domain(domain, role);
+    @Override
+    public String toString() {
+      return type.name().toLowerCase() + "-" + value();
+    }
+
+    static Entity fromPb(String entity) {
+      if (entity.startsWith("user-")) {
+        return new User(entity.substring(5));
+      }
+      if (entity.equals(User.ALL_USERS)) {
+        return User.ofAllUsers();
+      }
+      if (entity.equals(User.ALL_AUTHENTICATED_USERS)) {
+        return User.ofAllAuthenticatedUsers();
+      }
+      if (entity.startsWith("group-")) {
+        return new Group(entity.substring(6));
+      }
+      if (entity.startsWith("project-")) {
+        int idx = entity.indexOf('-', 9);
+        String team = entity.substring(9, idx);
+        String projectId = entity.substring(idx + 1);
+        return new Project(Project.ProjectRole.valueOf(team.toUpperCase()), projectId);
+      }
+      return null;
     }
   }
 
-  public static class Group extends Acl {
+  public static class Domain extends Entity {
 
-    private static final long serialVersionUID = -1660987136294408826L;
-    private final String email;
+    private static final long serialVersionUID = -3033025857280447253L;
+
+    public Domain(String domain) {
+      super(EntityType.DOMAIN, domain);
 
-    Group(String email, Role role) {
-      super(Type.GROUP, role);
-      this.email = email;
     }
 
-    public String email() {
-      return email;
+    public String domain() {
+      return value();
     }
+  }
+
+  public static class Group extends Entity {
+
+    private static final long serialVersionUID = -1660987136294408826L;
 
-    String entity() {
-      return "group-" + email;
+    public Group(String email) {
+      super(EntityType.GROUP, email);
     }
 
-    public static Group of(String email, Role role) {
-      return new Group(email, role);
+    public String email() {
+      return value();
     }
   }
 
-  public static class User extends Acl {
+  public static class User extends Entity {
 
     private static final long serialVersionUID = 3076518036392737008L;
     private static final String ALL_USERS = "allUsers";
     private static final String ALL_AUTHENTICATED_USERS = "allAuthenticatedUsers";
 
-    private final String email;
-
-    User(String email, Role role) {
-      super(Type.USER, role);
-      this.email = email;
+    public User(String email) {
+      super(EntityType.USER, email);
     }
 
     public String email() {
-      return email;
+      return value();
     }
 
-    String entity() {
-      switch (email) {
+    @Override
+    public String toString() {
+      switch (value()) {
         case ALL_AUTHENTICATED_USERS:
           return ALL_AUTHENTICATED_USERS;
         case ALL_USERS:
           return ALL_USERS;
         default:
-          return "user-" + email;
+          break;
       }
+      return super.toString();
     }
 
 
-    public static User of(String email, Role role) {
-      return new User(email, role);
+    public static User ofAllUsers() {
+      return new User(ALL_USERS);
     }
 
-    public static User ofAllUsers(Role role) {
-      return of(ALL_USERS, role);
-    }
-
-    public static User ofAllAuthenticatedUsers(Role role) {
-      return of(ALL_AUTHENTICATED_USERS, role);
+    public static User ofAllAuthenticatedUsers() {
+      return new User(ALL_AUTHENTICATED_USERS);
     }
   }
 
-  public static class Project extends Acl {
+  public static class Project extends Entity {
 
-    private final String projectId;
     private final ProjectRole pRole;
+    private final String projectId;
 
     enum ProjectRole {
       OWNERS,
@@ -141,8 +163,8 @@ enum ProjectRole {
       VIEWERS
     }
 
-    Project(ProjectRole pRole, String projectId, Role role) {
-      super(Type.PROJECT, role);
+    Project(ProjectRole pRole, String projectId) {
+      super(EntityType.PROJECT, pRole.name().toLowerCase() + "-" + projectId);
       this.pRole = pRole;
       this.projectId = projectId;
     }
@@ -154,23 +176,15 @@ public ProjectRole projectRole() {
     public String projectId() {
       return projectId;
     }
-
-    String entity() {
-      return "project-" + pRole.name().toLowerCase() + "-" + projectId;
-    }
-
-    public static Project of(ProjectRole pRole, String projectId, Role role) {
-      return new Project(pRole, projectId, role);
-    }
   }
 
-  Acl(Type type, Role role) {
-    this.type = type;
+  public Acl(Entity entity, Role role) {
+    this.entity = entity;
     this.role = role;
   }
 
-  public Type type() {
-    return type;
+  public Entity entity() {
+    return entity;
   }
 
   public Role role() {
@@ -180,48 +194,24 @@ public Role role() {
   BucketAccessControl toBucketPb() {
     BucketAccessControl bucketPb = new BucketAccessControl();
     bucketPb.setRole(role().toString());
-    bucketPb.setEntity(entity());
+    bucketPb.setEntity(entity().toString());
     return bucketPb;
   }
 
   ObjectAccessControl toObjectPb() {
     ObjectAccessControl objectPb = new ObjectAccessControl();
     objectPb.setRole(role().toString());
-    objectPb.setEntity(entity());
+    objectPb.setEntity(entity().toString());
     return objectPb;
   }
 
-  abstract String entity();
-
   static Acl fromPb(ObjectAccessControl objectAccessControl) {
     Role role = Role.valueOf(objectAccessControl.getRole());
-    return forEntity(objectAccessControl.getEntity(), role);
+    return new Acl(Entity.fromPb(objectAccessControl.getEntity()), role);
   }
 
   static Acl fromPb(BucketAccessControl bucketAccessControl) {
     Role role = Role.valueOf(bucketAccessControl.getRole());
-    return forEntity(bucketAccessControl.getEntity(), role);
-  }
-
-  private static Acl forEntity(String entity, Role role) {
-    if (entity.startsWith("user-")) {
-      return User.of(entity.substring(5), role);
-    }
-    if (entity.equals(User.ALL_USERS)) {
-      return User.ofAllUsers(role);
-    }
-    if (entity.equals(User.ALL_AUTHENTICATED_USERS)) {
-      return User.ofAllAuthenticatedUsers(role);
-    }
-    if (entity.startsWith("group-")) {
-      return Group.of(entity.substring(6), role);
-    }
-    if (entity.startsWith("project-")) {
-      int idx = entity.indexOf('-', 9);
-      String team = entity.substring(9, idx);
-      String projectId = entity.substring(idx + 1);
-      return Project.of(ProjectRole.valueOf(team.toUpperCase()), projectId, role);
-    }
-    return null;
+    return new Acl(Entity.fromPb(bucketAccessControl.getEntity()), role);
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index 7f6de0300180..085317e41961 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -40,7 +40,7 @@ public final class BucketInfo implements Serializable {
 
   private final String id;
   private final String name;
-  private final String owner;
+  private final Acl.Entity owner;
   private final String selfLink;
   private final boolean versioningEnabled;
   private final String etag;

From 9dbd7c957205b26c6b601be8f812898e33745ec4 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 10 Apr 2015 16:23:56 -0700
Subject: [PATCH 162/732] work in progress

---
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../java/com/google/gcloud/storage/Acl.java   |  56 ++--
 .../com/google/gcloud/storage/BucketInfo.java | 290 ++++++++++++++++--
 3 files changed, 304 insertions(+), 44 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index c53574986c4b..d4e6a245dea5 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
  void patch(Bucket bucket) throws IOException;
  void patch(StorageObject storageObject) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index d2e2be0848f1..48bd6c787718 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -34,25 +34,26 @@ public enum Role {
     WRITER
   }
 
-  public enum EntityType {
-    DOMAIN,
-    GROUP,
-    USER,
-    PROJECT
-  }
-
   public static abstract class Entity implements Serializable {
 
     private static final long serialVersionUID = -2707407252771255840L;
-    private final EntityType type;
+    private final Type type;
     private final String value;
 
-    Entity(EntityType type, String value) {
+    public enum Type {
+      DOMAIN,
+      GROUP,
+      USER,
+      PROJECT,
+      UNKNOWN
+    }
+
+    Entity(Type type, String value) {
       this.type = type;
       this.value = value;
     }
 
-    public EntityType type() {
+    public Type type() {
       return type;
     }
 
@@ -62,6 +63,10 @@ protected String value() {
 
     @Override
     public String toString() {
+      return toPb();
+    }
+
+    String toPb() {
       return type.name().toLowerCase() + "-" + value();
     }
 
@@ -84,7 +89,7 @@ static Entity fromPb(String entity) {
         String projectId = entity.substring(idx + 1);
         return new Project(Project.ProjectRole.valueOf(team.toUpperCase()), projectId);
       }
-      return null;
+      return new RawEntity(entity);
     }
   }
 
@@ -93,7 +98,7 @@ public static class Domain extends Entity {
     private static final long serialVersionUID = -3033025857280447253L;
 
     public Domain(String domain) {
-      super(EntityType.DOMAIN, domain);
+      super(Type.DOMAIN, domain);
 
     }
 
@@ -107,7 +112,7 @@ public static class Group extends Entity {
     private static final long serialVersionUID = -1660987136294408826L;
 
     public Group(String email) {
-      super(EntityType.GROUP, email);
+      super(Type.GROUP, email);
     }
 
     public String email() {
@@ -122,7 +127,7 @@ public static class User extends Entity {
     private static final String ALL_AUTHENTICATED_USERS = "allAuthenticatedUsers";
 
     public User(String email) {
-      super(EntityType.USER, email);
+      super(Type.USER, email);
     }
 
     public String email() {
@@ -130,7 +135,7 @@ public String email() {
     }
 
     @Override
-    public String toString() {
+    String toPb() {
       switch (value()) {
         case ALL_AUTHENTICATED_USERS:
           return ALL_AUTHENTICATED_USERS;
@@ -139,10 +144,9 @@ public String toString() {
         default:
           break;
       }
-      return super.toString();
+      return super.toPb();
     }
 
-
     public static User ofAllUsers() {
       return new User(ALL_USERS);
     }
@@ -164,7 +168,7 @@ enum ProjectRole {
     }
 
     Project(ProjectRole pRole, String projectId) {
-      super(EntityType.PROJECT, pRole.name().toLowerCase() + "-" + projectId);
+      super(Type.PROJECT, pRole.name().toLowerCase() + "-" + projectId);
       this.pRole = pRole;
       this.projectId = projectId;
     }
@@ -178,6 +182,18 @@ public String projectId() {
     }
   }
 
+  public static class RawEntity extends Entity {
+
+    RawEntity(String entity) {
+      super(Type.UNKNOWN, entity);
+    }
+
+    @Override
+    String toPb() {
+      return value();
+    }
+  }
+
   public Acl(Entity entity, Role role) {
     this.entity = entity;
     this.role = role;
@@ -200,8 +216,8 @@ BucketAccessControl toBucketPb() {
 
   ObjectAccessControl toObjectPb() {
     ObjectAccessControl objectPb = new ObjectAccessControl();
-    objectPb.setRole(role().toString());
-    objectPb.setEntity(entity().toString());
+    objectPb.setRole(role().name());
+    objectPb.setEntity(entity().toPb());
     return objectPb;
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/BucketInfo.java
index 085317e41961..97f1bea1e244 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -18,18 +18,19 @@
 
 import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.collect.Iterables.transform;
-import static com.google.common.collect.Lists.newArrayList;
 
 import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.Bucket;
-import com.google.api.services.storage.model.Bucket.Lifecycle;
+import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
+import com.google.api.services.storage.model.Bucket.Versioning;
 import com.google.api.services.storage.model.BucketAccessControl;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.gcloud.storage.Acl.Entity;
 
 import java.io.Serializable;
 import java.util.List;
@@ -43,6 +44,9 @@ public final class BucketInfo implements Serializable {
   private final Acl.Entity owner;
   private final String selfLink;
   private final boolean versioningEnabled;
+  private final String indexPage;
+  private final String notFoundPage;
+  private final ImmutableList deleteRules;
   private final String etag;
   private final long createTime;
   private final long metageneration;
@@ -53,13 +57,151 @@ public final class BucketInfo implements Serializable {
   private final StorageClass storageClass;
 
 
-  public static abstract class AutoDeleteRule implements Serializable {
+  public static abstract class DeleteRule implements Serializable {
 
-    public enum Option {
-      AGE, CREATE_BEFORE, VERSION_LIMIT, NOT_LIVE
+    private static final long serialVersionUID = 3137971668395933033L;
+    private final Type type;
+    private static final String SUPPORTED_ACTION = "Delete";
+
+    public enum Type {
+      AGE, CREATE_BEFORE, NUM_NEWER_VERSIONS, IS_LIVE, UNKNOWN
+    }
+
+    DeleteRule(Type type) {
+      this.type = type;
+    }
+
+    public Type type() {
+      return type;
+    }
+
+    Rule toPb() {
+      Rule rule = new Rule();
+      rule.setAction(new Rule.Action().setType(SUPPORTED_ACTION));
+      Rule.Condition condition = new Rule.Condition();
+      populateCondition(condition);
+      rule.setCondition(condition);
+      return rule;
+    }
+
+    abstract void populateCondition(Rule.Condition condition);
+
+    static DeleteRule fromPb(Rule rule) {
+      if (rule.getAction() != null && SUPPORTED_ACTION.endsWith(rule.getAction().getType())) {
+        Rule.Condition condition = rule.getCondition();
+        Integer age = condition.getAge();
+        if (age != null) {
+          return new AgeDeleteRule(age);
+        }
+        DateTime dateTime = condition.getCreatedBefore();
+        if (dateTime != null) {
+          return new CreatedBeforeDeleteRule(dateTime.getValue());
+        }
+        Integer numNewerVersions = condition.getNumNewerVersions();
+        if (numNewerVersions != null) {
+          return new NumNewerVersionsDeleteRule(numNewerVersions);
+        }
+        Boolean isLive = condition.getIsLive();
+        if (isLive != null) {
+          return new IsLiveDeleteRule(isLive);
+        }
+      }
+      return new RawDeleteRule(rule);
+    }
+  }
+
+  public static class AgeDeleteRule extends DeleteRule {
+
+    private static final long serialVersionUID = 5697166940712116380L;
+    private final int daysToLive;
+
+    public AgeDeleteRule(int daysToLive) {
+      super(Type.AGE);
+      this.daysToLive = daysToLive;
+    }
+
+    public int daysToLive() {
+      return daysToLive;
+    }
+
+    void populateCondition(Rule.Condition condition) {
+      condition.setAge(daysToLive);
+    }
+  }
+
+  static class RawDeleteRule extends DeleteRule {
+
+    private final Rule rule;
+
+    RawDeleteRule(Rule rule) {
+      super(Type.UNKNOWN);
+      this.rule = rule;
+    }
+
+    void populateCondition(Rule.Condition condition) {
+      throw new UnsupportedOperationException();
+    }
+
+    Rule toPb() {
+      return rule;
     }
+  }
+
+  public static class CreatedBeforeDeleteRule extends DeleteRule {
 
+    private static final long serialVersionUID = 881692650279195867L;
+    private final long timeMillis;
 
+    public CreatedBeforeDeleteRule(long timeMillis) {
+      super(Type.CREATE_BEFORE);
+      this.timeMillis = timeMillis;
+    }
+
+    public long timeMillis() {
+      return timeMillis;
+    }
+
+    void populateCondition(Rule.Condition condition) {
+      condition.setCreatedBefore(new DateTime(timeMillis));
+    }
+  }
+
+  public static class NumNewerVersionsDeleteRule extends DeleteRule {
+
+    private static final long serialVersionUID = -1955554976528303894L;
+    private final int numNewerVersions;
+
+    public NumNewerVersionsDeleteRule(int numNewerVersions) {
+      super(Type.NUM_NEWER_VERSIONS);
+      this.numNewerVersions = numNewerVersions;
+    }
+
+    public int numNewerVersions() {
+      return numNewerVersions;
+    }
+
+    void populateCondition(Rule.Condition condition) {
+      condition.setNumNewerVersions(numNewerVersions);
+    }
+  }
+
+  public static class IsLiveDeleteRule extends DeleteRule {
+
+    private static final long serialVersionUID = -3502994563121313364L;
+    private final boolean isLive;
+
+    public IsLiveDeleteRule(boolean isLive) {
+      super(Type.IS_LIVE);
+      this.isLive = isLive;
+    }
+
+    public boolean isLive() {
+      return isLive;
+    }
+
+    void populateCondition(Rule.Condition condition) {
+      condition.setIsLive(isLive);
+    }
   }
 
   public static final class StorageClass implements Serializable {
@@ -173,6 +315,12 @@ public final static class Builder {
 
     private final String id;
     private final String name;
+    private Acl.Entity owner;
+    private String selfLink;
+    private boolean versioningEnabled;
+    private String indexPage;
+    private String notFoundPage;
+    private ImmutableList deleteRules = ImmutableList.of();
     private StorageClass storageClass;
     private Location location;
     private String etag;
@@ -187,6 +335,36 @@ public final static class Builder {
       this.name = name;
     }
 
+    Builder owner(Acl.Entity owner) {
+      this.owner = owner;
+      return this;
+    }
+
+    Builder selfLink(String selfLink) {
+      this.selfLink = selfLink;
+      return this;
+    }
+
+    public Builder versioningEnabled(boolean enable) {
+      this.versioningEnabled = enable;
+      return this;
+    }
+
+    public Builder indexPage(String indexPage) {
+      this.indexPage = indexPage;
+      return this;
+    }
+
+    public Builder notFoundPage(String notFoundPage) {
+      this.notFoundPage = notFoundPage;
+      return this;
+    }
+
+    public Builder deleteRules(Iterable rules) {
+      this.deleteRules = ImmutableList.copyOf(rules);
+      return this;
+    }
+
     public Builder storageClass(StorageClass storageClass) {
       this.storageClass = storageClass;
       return this;
@@ -243,6 +421,12 @@ private BucketInfo(Builder builder) {
     cors = ImmutableList.copyOf(builder.cors);
     acl = ImmutableList.copyOf(builder.acl);
     defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
+    owner = builder.owner;
+    selfLink = builder.selfLink;
+    versioningEnabled = builder.versioningEnabled;
+    indexPage = builder.indexPage;
+    notFoundPage = builder.notFoundPage;
+    deleteRules = ImmutableList.copyOf(builder.deleteRules);
   }
 
   public String id() {
@@ -253,6 +437,30 @@ public String name() {
     return name;
   }
 
+  public Entity Owner() {
+    return owner;
+  }
+
+  public String selfLink() {
+    return selfLink;
+  }
+
+  public boolean versioningEnabled() {
+    return versioningEnabled;
+  }
+
+  public String indexPage() {
+    return indexPage;
+  }
+
+  public String notFoundPage() {
+    return notFoundPage;
+  }
+
+  public ImmutableList deleteRules() {
+    return deleteRules;
+  }
+
   public String etag() {
     return etag;
   }
@@ -294,7 +502,13 @@ public Builder toBuilder() {
         .acl(acl)
         .defaultAcl(defaultAcl)
         .location(location)
-        .storageClass(storageClass);
+        .storageClass(storageClass)
+        .owner(owner)
+        .selfLink(selfLink)
+        .versioningEnabled(versioningEnabled)
+        .indexPage(indexPage)
+        .notFoundPage(notFoundPage)
+        .deleteRules(deleteRules);
   }
 
   BucketInfo fromPb(Bucket bucket) {
@@ -318,13 +532,27 @@ BucketInfo fromPb(Bucket bucket) {
           @Override public Acl apply(ObjectAccessControl objectAccessControl) {
             return Acl.fromPb(objectAccessControl);
           }
-        }));
-    bucket.setOwner();
-    bucket.setSelfLink();
-    bucket.setWebsite();
-    bucket.setVersioning();
-    bucket.setLifecycle(new Lifecycle.Rule().
-
+        }))
+        .owner(Entity.fromPb(bucket.getOwner().getEntity()))
+        .selfLink(bucket.getSelfLink());
+    Bucket.Versioning versioning = bucket.getVersioning();
+    if (versioning != null) {
+      builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE));
+    }
+    Bucket.Website website = bucket.getWebsite();
+    if (website != null) {
+      builder.indexPage(website.getMainPageSuffix());
+      builder.notFoundPage(website.getNotFoundPage());
+    }
+    Bucket.Lifecycle lifecycle = bucket.getLifecycle();
+    if (lifecycle != null) {
+      builder.deleteRules(transform(lifecycle.getRule(),
+          new Function() {
+            @Override public DeleteRule apply(Rule rule) {
+              return DeleteRule.fromPb(rule);
+            }
+          }));
+    }
     return builder.build();
   }
 
@@ -345,22 +573,38 @@ Bucket toPb() {
     if (storageClass != null) {
       bucket.setStorageClass(storageClass.value());
     }
-    bucket.setCors(newArrayList(Iterables.transform(cors, new Function() {
-      @Override public Bucket.Cors apply(Cors cors) {
+    bucket.setCors(Lists.transform(cors, new Function() {
+      @Override
+      public Bucket.Cors apply(Cors cors) {
         return cors.toPb();
       }
-    })));
-    bucket.setAcl(newArrayList(Iterables.transform(acl, new Function() {
-      @Override public BucketAccessControl apply(Acl acl) {
+    }));
+    bucket.setAcl(Lists.transform(acl, new Function() {
+      @Override
+      public BucketAccessControl apply(Acl acl) {
         return acl.toBucketPb();
       }
-    })));
+    }));
     bucket.setDefaultObjectAcl(
-        newArrayList(Iterables.transform(defaultAcl, new Function() {
-          @Override public ObjectAccessControl apply(Acl acl) {
+        Lists.transform(defaultAcl, new Function() {
+          @Override
+          public ObjectAccessControl apply(Acl acl) {
             return acl.toObjectPb();
           }
-        })));
+        }));
+    bucket.setOwner(new Bucket.Owner().setEntity(owner.toPb()));
+    bucket.setSelfLink(selfLink);
+    bucket.setVersioning(new Versioning().setEnabled(versioningEnabled));
+    Bucket.Website website = new Bucket.Website();
+    website.setMainPageSuffix(indexPage);
+    website.setNotFoundPage(notFoundPage);
+    bucket.setWebsite(website);
+    Bucket.Lifecycle lifecycle = new Bucket.Lifecycle();
+    lifecycle.setRule(Lists.transform(deleteRules, new Function() {
+      @Override public Rule apply(DeleteRule deleteRule) {
+        return deleteRule.toPb();
+      }
+    }));
     return bucket;
   }
 }

From 3b8f4e36c1aae0293f4277d26fb62f36e7c45b39 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sat, 11 Apr 2015 12:42:21 -0700
Subject: [PATCH 163/732] work in progress

---
 .../com/google/gcloud/storage/ObjectInfo.java | 141 ++++++++++++++----
 1 file changed, 108 insertions(+), 33 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
index 05a0c901284a..2e96a9163e93 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java
+++ b/src/main/java/com/google/gcloud/storage/ObjectInfo.java
@@ -34,13 +34,14 @@ public class ObjectInfo implements Serializable {
   private static final long serialVersionUID = 2228487739943277159L;
 
   private final String bucket;
+  private final String id;
   private final String name;
-  private final String contentType;
+  private final String selfLink;
   private final String cacheControl;
   private final ImmutableList acl;
-  private final String owner;
+  private final Acl.Entity owner;
   private final long size;
-  private final String contentEncoding;
+  private final String etag;
   private final String md5;
   private final String crc32c;
   private final String mediaLink;
@@ -49,17 +50,28 @@ public class ObjectInfo implements Serializable {
   private final long metageneration;
   private final long deleteTime;
   private final long updateTime;
+  private final String contentType;
+  private final String contentEncoding;
+  private final String contentDisposition;
+  private final String contentLanguage;
+  private final int componentCount;
 
   public static final class Builder {
 
     private String bucket;
+    private String id;
     private String name;
     private String contentType;
+    private String contentEncoding;
+    private String contentDisposition;
+    private  String contentLanguage;
+    private int componentCount;
     private String cacheControl;
     private ImmutableList acl;
-    private String owner;
+    private Acl.Entity owner;
     private long size;
-    private String contentEncoding;
+    private String etag;
+    private String selfLink;
     private String md5;
     private String crc32c;
     private String mediaLink;
@@ -77,6 +89,11 @@ public Builder bucket(String bucket) {
       return this;
     }
 
+    Builder id(String id) {
+      this.id = id;
+      return this;
+    }
+
     public Builder name(String name) {
       this.name = name;
       return this;
@@ -87,6 +104,26 @@ public Builder contentType(String contentType) {
       return this;
     }
 
+    Builder contentDisposition(String contentDisposition) {
+      this.contentDisposition = contentDisposition;
+      return this;
+    }
+
+    Builder contentLanguage(String contentLanguage) {
+      this.contentLanguage = contentLanguage;
+      return this;
+    }
+
+    public Builder contentEncoding(String contentEncoding) {
+      this.contentEncoding = contentEncoding;
+      return this;
+    }
+
+    Builder componentCount(int componentCount) {
+      this.componentCount = componentCount;
+      return this;
+    }
+
     public Builder cacheControl(String cacheControl) {
       this.cacheControl = cacheControl;
       return this;
@@ -97,7 +134,7 @@ public Builder acl(List acl) {
       return this;
     }
 
-    public Builder owner(String owner) {
+    public Builder owner(Acl.Entity owner) {
       this.owner = owner;
       return this;
     }
@@ -107,8 +144,13 @@ public Builder size(long size) {
       return this;
     }
 
-    public Builder contentEncoding(String contentEncoding) {
-      this.contentEncoding = contentEncoding;
+    Builder etag(String etag) {
+      this.etag = etag;
+      return this;
+    }
+
+    Builder selfLink(String selfLink) {
+      this.selfLink = selfLink;
       return this;
     }
 
@@ -159,13 +201,19 @@ public ObjectInfo build() {
 
   private ObjectInfo(Builder builder) {
     bucket = builder.bucket;
+    id = builder.id;
     name = builder.name;
-    contentType = builder.contentType;
     cacheControl = builder.cacheControl;
+    contentEncoding = builder.contentEncoding;
+    contentType = builder.contentType;
+    contentDisposition = builder.contentDisposition;
+    contentLanguage = builder.contentLanguage;
+    componentCount = builder.componentCount;
     acl = builder.acl;
     owner = builder.owner;
     size = builder.size;
-    contentEncoding = builder.contentEncoding;
+    etag = builder.etag;
+    selfLink = builder.selfLink;
     md5 = builder.md5;
     crc32c = builder.crc32c;
     mediaLink = builder.mediaLink;
@@ -180,12 +228,12 @@ public String bucket() {
     return bucket;
   }
 
-  public String name() {
-    return name;
+  public String id() {
+    return id;
   }
 
-  public String contentType() {
-    return contentType;
+  public String name() {
+    return name;
   }
 
   public String cacheControl() {
@@ -196,7 +244,7 @@ public List acl() {
     return acl;
   }
 
-  public String owner() {
+  public Acl.Entity owner() {
     return owner;
   }
 
@@ -204,10 +252,34 @@ public long size() {
     return size;
   }
 
+  public String contentType() {
+    return contentType;
+  }
+
   public String contentEncoding() {
     return contentEncoding;
   }
 
+  public String contentDisposition() {
+    return contentDisposition;
+  }
+
+  public String contentLanguage() {
+    return contentEncoding;
+  }
+
+  public int componentCount() {
+    return componentCount;
+  }
+
+  public String etag() {
+    return etag;
+  }
+
+  public String selfLink() {
+    return selfLink;
+  }
+
   public String md5() {
     return md5;
   }
@@ -257,8 +329,13 @@ public Builder toBuilder() {
         .name(name)
         .owner(owner)
         .updateTime(updateTime)
-        .size(size);
-
+        .size(size)
+        .contentDisposition(contentDisposition)
+        .componentCount(componentCount)
+        .contentLanguage(contentLanguage)
+        .etag(etag)
+        .id(id)
+        .selfLink(selfLink);
   }
 
   public static Builder builder() {
@@ -286,16 +363,15 @@ public ObjectAccessControl apply(Acl acl) {
     storageObject.setMetadata(metadata);
     storageObject.setMetageneration(metageneration);
     storageObject.setName(name);
-    storageObject.setOwner(owner);
+    storageObject.setOwner(new StorageObject.Owner().setEntity(owner.toPb()));
     storageObject.setUpdated(new DateTime(updateTime));
     storageObject.setSize(BigInteger.valueOf(size));
-    storageObject.setContentDisposition();
-    storageObject.setComponentCount();
-    storageObject.setContentLanguage();
-    storageObject.setEtag();
-    storageObject.setId();
-    storageObject.setSelfLink();
-    storageObject.setStorageClass();
+    storageObject.setContentDisposition(contentDisposition);
+    storageObject.setComponentCount(componentCount);
+    storageObject.setContentLanguage(contentLanguage);
+    storageObject.setEtag(etag);
+    storageObject.setId(id);
+    storageObject.setSelfLink(selfLink);
     return storageObject;
   }
 
@@ -319,16 +395,15 @@ static ObjectInfo fromPb(StorageObject storageObject) {
         .metadata(storageObject.getMetadata())
         .metageneration(storageObject.getMetageneration())
         .name(storageObject.getName())
-        .owner(storageObject.getName())
+        .owner(Acl.Entity.fromPb(storageObject.getOwner().getEntity()))
         .updateTime(storageObject.getUpdated().getValue())
         .size(storageObject.getSize().longValue())
+        .contentDisposition(storageObject.getContentDisposition())
+        .componentCount(storageObject.getComponentCount())
+        .contentLanguage(storageObject.getContentLanguage())
+        .etag(storageObject.getEtag())
+        .id(storageObject.getId())
+        .selfLink(storageObject.getSelfLink())
         .build();
-    storageObject.setContentDisposition();
-    storageObject.setComponentCount();
-    storageObject.setContentLanguage();
-    storageObject.setEtag();
-    storageObject.setId();
-    storageObject.setSelfLink();
-    storageObject.setStorageClass();
   }
 }

From f94246cf0db7bd4a4e1937a805bc51922910f414 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 13 Apr 2015 17:41:04 -0700
Subject: [PATCH 164/732] work in progress

---
 .../gcloud/examples/StorageExample.java       |  16 +-
 .../google/gcloud/spi/DefaultStorageRpc.java  |   2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../storage/{ObjectInfo.java => Blob.java}    |  16 +-
 .../storage/{BucketInfo.java => Bucket.java}  | 169 +++++++++---------
 .../gcloud/storage/ObjectWriteChannel.java    |   2 +
 .../google/gcloud/storage/StorageService.java |  23 +--
 .../gcloud/storage/StorageServiceImpl.java    |  58 ++++--
 8 files changed, 164 insertions(+), 124 deletions(-)
 rename src/main/java/com/google/gcloud/storage/{ObjectInfo.java => Blob.java} (96%)
 rename src/main/java/com/google/gcloud/storage/{BucketInfo.java => Bucket.java} (80%)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 87bf24d28772..ebfbeef3545f 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,8 +16,7 @@
 
 package com.google.gcloud.examples;
 
-import com.google.api.services.storage.Storage;
-import com.google.gcloud.storage.BucketInfo;
+import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.StorageService;
 import com.google.gcloud.storage.StorageServiceFactory;
 import com.google.gcloud.storage.StorageServiceOptions;
@@ -47,7 +46,7 @@ public class StorageExample {
 
   private static abstract class StorageAction {
 
-    abstract void run(StorageService storage, BucketInfo bucket, String folder, String... args);
+    abstract void run(StorageService storage, Bucket bucket, String folder, String... args);
 
     protected String getRequiredParams() {
       return "";
@@ -65,7 +64,7 @@ protected String fullPath(String folder, String file) {
 
   private static class DeleteAction extends StorageAction {
     @Override
-    public void run(StorageService storage, BucketInfo bucket, String folder, String... args) {
+    public void run(StorageService storage, Bucket bucket, String folder, String... args) {
       for (String file : args) {
         storage.delete(bucket.name(), fullPath(folder, file));
       }
@@ -84,10 +83,10 @@ public String getRequiredParams() {
   public static void main(String... args) {
     StorageAction action = null;
     StorageService storage = null;
-    BucketInfo bucketInfo = null;
+    Bucket bucket = null;
     String folder = DEFAULT_FOLDER;
     if (args.length > 0) {
-      String bucket = args[0];
+      String bucketName = args[0];
       String actionName = DEFAULT_ACTION;
       if (args.length > 1) {
         folder = args[1];
@@ -95,7 +94,7 @@ public static void main(String... args) {
           actionName = args[2].toLowerCase();
         }
         storage = StorageServiceFactory.getDefault(StorageServiceOptions.builder().build());
-        bucketInfo = storage.get(bucket);
+        bucket = storage.get(bucketName);
       }
       action = ACTIONS.get(actionName);
     }
@@ -117,7 +116,6 @@ public static void main(String... args) {
     }
 
     args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{};
-    action.run(bucketInfo, folder, args);
-    }
+    action.run(storage, bucket, folder, args);
   }
 }
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 3295fa003530..44dc5df4e86a 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public Bucket bucket(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public List objects(String bucket, String prefix, String delimiter)
      throws IOException {
    return null;
  }

  @Override
  public Bucket get(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }

  @Override
  public StorageObject get(String bucket, String object) throws IOException {
    return storage.objects().get(bucket, object).execute();
  }

  @Override
  public Bucket patch(Bucket bucket) throws IOException {
    return storage.buckets().patch(bucket.getName(), bucket).execute();
  }

  @Override
  public StorageObject patch(StorageObject storageObject) throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
  }

  @Override
  public void delete(String bucket, String object) throws IOException {
    storage.objects().delete(bucket, object).execute();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index d4e6a245dea5..9c650f5113fe 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  Bucket bucket(String name) throws IOException;
  void patch(Bucket bucket) throws IOException;
  void patch(StorageObject storageObject) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  List objects(String bucket, String prefix, String delimiter) throws IOException;
  Bucket get(String bucket) throws IOException;
  StorageObject get(String bucket, String object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  //void delete(String bucket) throws  IOException;
  void delete(String bucket, String object) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/ObjectInfo.java b/src/main/java/com/google/gcloud/storage/Blob.java
similarity index 96%
rename from src/main/java/com/google/gcloud/storage/ObjectInfo.java
rename to src/main/java/com/google/gcloud/storage/Blob.java
index 2e96a9163e93..e5fb3a76a6e2 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectInfo.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -19,6 +19,7 @@
 import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.api.services.storage.model.StorageObject;
+import com.google.api.services.storage.model.StorageObject.Owner;
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -29,7 +30,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class ObjectInfo implements Serializable {
+public class Blob implements Serializable {
 
   private static final long serialVersionUID = 2228487739943277159L;
 
@@ -194,12 +195,12 @@ public Builder updateTime(long updateTime) {
       return this;
     }
 
-    public ObjectInfo build() {
-      return new ObjectInfo(this);
+    public Blob build() {
+      return new Blob(this);
     }
   }
 
-  private ObjectInfo(Builder builder) {
+  private Blob(Builder builder) {
     bucket = builder.bucket;
     id = builder.id;
     name = builder.name;
@@ -346,8 +347,7 @@ StorageObject toPb() {
     StorageObject storageObject = new StorageObject();
     storageObject.setAcl(Lists.transform(acl,
         new Function() {
-          @Override
-          public ObjectAccessControl apply(Acl acl) {
+          @Override public ObjectAccessControl apply(Acl acl) {
             return acl.toObjectPb();
           }
         }));
@@ -363,7 +363,7 @@ public ObjectAccessControl apply(Acl acl) {
     storageObject.setMetadata(metadata);
     storageObject.setMetageneration(metageneration);
     storageObject.setName(name);
-    storageObject.setOwner(new StorageObject.Owner().setEntity(owner.toPb()));
+    storageObject.setOwner(new Owner().setEntity(owner.toPb()));
     storageObject.setUpdated(new DateTime(updateTime));
     storageObject.setSize(BigInteger.valueOf(size));
     storageObject.setContentDisposition(contentDisposition);
@@ -375,7 +375,7 @@ public ObjectAccessControl apply(Acl acl) {
     return storageObject;
   }
 
-  static ObjectInfo fromPb(StorageObject storageObject) {
+  static Blob fromPb(StorageObject storageObject) {
     return builder()
         .acl(Lists.transform(storageObject.getAcl(),
             new Function() {
diff --git a/src/main/java/com/google/gcloud/storage/BucketInfo.java b/src/main/java/com/google/gcloud/storage/Bucket.java
similarity index 80%
rename from src/main/java/com/google/gcloud/storage/BucketInfo.java
rename to src/main/java/com/google/gcloud/storage/Bucket.java
index 97f1bea1e244..7d3944c0d9c0 100644
--- a/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -17,25 +17,26 @@
 package com.google.gcloud.storage;
 
 import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.collect.Iterables.transform;
+import static com.google.common.collect.Lists.transform;
 
 import com.google.api.client.util.DateTime;
-import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.Bucket.Lifecycle;
 import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
+import com.google.api.services.storage.model.Bucket.Owner;
 import com.google.api.services.storage.model.Bucket.Versioning;
+import com.google.api.services.storage.model.Bucket.Website;
 import com.google.api.services.storage.model.BucketAccessControl;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
 import com.google.gcloud.storage.Acl.Entity;
 
 import java.io.Serializable;
 import java.util.List;
 
-public final class BucketInfo implements Serializable {
+public final class Bucket implements Serializable {
 
   private static final long serialVersionUID = -3946094202176916586L;
 
@@ -390,7 +391,7 @@ Builder metageneration(Long metageneration) {
       return this;
     }
 
-    Builder cors(Iterable cors) {
+    public Builder cors(Iterable cors) {
       this.cors = cors;
       return this;
     }
@@ -405,12 +406,12 @@ public Builder defaultAcl(Iterable acl) {
       return this;
     }
 
-    public BucketInfo build() {
-      return new BucketInfo(this);
+    public Bucket build() {
+      return new Bucket(this);
     }
   }
 
-  private BucketInfo(Builder builder) {
+  private Bucket(Builder builder) {
     id = builder.id;
     name = builder.name;
     etag = builder.etag;
@@ -511,100 +512,102 @@ public Builder toBuilder() {
         .deleteRules(deleteRules);
   }
 
-  BucketInfo fromPb(Bucket bucket) {
-    Builder builder = new Builder(bucket.getId(), bucket.getName())
-        .createTime(bucket.getTimeCreated().getValue())
-        .etag(bucket.getEtag())
-        .metageneration(bucket.getMetageneration())
-        .location(Location.of(bucket.getLocation()))
-        .storageClass(StorageClass.of(bucket.getStorageClass()))
-        .cors(transform(bucket.getCors(), new Function() {
-          @Override public Cors apply(Bucket.Cors cors) {
-            return Cors.fromPb(cors);
-          }
-        }))
-        .acl(transform(bucket.getAcl(), new Function() {
-          @Override public Acl apply(BucketAccessControl bucketAccessControl) {
-            return Acl.fromPb(bucketAccessControl);
-          }
-        }))
-        .defaultAcl(transform(bucket.getDefaultObjectAcl(), new Function() {
-          @Override public Acl apply(ObjectAccessControl objectAccessControl) {
-            return Acl.fromPb(objectAccessControl);
-          }
-        }))
-        .owner(Entity.fromPb(bucket.getOwner().getEntity()))
-        .selfLink(bucket.getSelfLink());
-    Bucket.Versioning versioning = bucket.getVersioning();
-    if (versioning != null) {
-      builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE));
-    }
-    Bucket.Website website = bucket.getWebsite();
-    if (website != null) {
-      builder.indexPage(website.getMainPageSuffix());
-      builder.notFoundPage(website.getNotFoundPage());
-    }
-    Bucket.Lifecycle lifecycle = bucket.getLifecycle();
-    if (lifecycle != null) {
-      builder.deleteRules(transform(lifecycle.getRule(),
-          new Function() {
-            @Override public DeleteRule apply(Rule rule) {
-              return DeleteRule.fromPb(rule);
-            }
-          }));
-    }
-    return builder.build();
-  }
-
-  Bucket toPb() {
-    Bucket bucket = new Bucket();
-    bucket.setId(id);
-    bucket.setName(name);
-    bucket.setEtag(etag);
+  com.google.api.services.storage.model.Bucket toPb() {
+    com.google.api.services.storage.model.Bucket bucketPb =
+        new com.google.api.services.storage.model.Bucket();
+    bucketPb.setId(id);
+    bucketPb.setName(name);
+    bucketPb.setEtag(etag);
     if (createTime > 0) {
-      bucket.setTimeCreated(new DateTime(createTime));
+      bucketPb.setTimeCreated(new DateTime(createTime));
     }
     if (metageneration > 0) {
-      bucket.setMetageneration(metageneration);
+      bucketPb.setMetageneration(metageneration);
     }
     if (location != null) {
-      bucket.setLocation(location.value());
+      bucketPb.setLocation(location.value());
     }
     if (storageClass != null) {
-      bucket.setStorageClass(storageClass.value());
+      bucketPb.setStorageClass(storageClass.value());
     }
-    bucket.setCors(Lists.transform(cors, new Function() {
-      @Override
-      public Bucket.Cors apply(Cors cors) {
-        return cors.toPb();
-      }
-    }));
-    bucket.setAcl(Lists.transform(acl, new Function() {
-      @Override
-      public BucketAccessControl apply(Acl acl) {
+    bucketPb.setCors(
+        transform(cors, new Function() {
+          @Override public com.google.api.services.storage.model.Bucket.Cors apply(Cors cors) {
+            return cors.toPb();
+          }
+        }));
+    bucketPb.setAcl(transform(acl, new Function() {
+      @Override public BucketAccessControl apply(Acl acl) {
         return acl.toBucketPb();
       }
     }));
-    bucket.setDefaultObjectAcl(
-        Lists.transform(defaultAcl, new Function() {
-          @Override
-          public ObjectAccessControl apply(Acl acl) {
+    bucketPb.setDefaultObjectAcl(transform(defaultAcl,
+        new Function() {
+          @Override public ObjectAccessControl apply(Acl acl) {
             return acl.toObjectPb();
           }
         }));
-    bucket.setOwner(new Bucket.Owner().setEntity(owner.toPb()));
-    bucket.setSelfLink(selfLink);
-    bucket.setVersioning(new Versioning().setEnabled(versioningEnabled));
-    Bucket.Website website = new Bucket.Website();
+    bucketPb.setOwner(new Owner().setEntity(owner.toPb()));
+    bucketPb.setSelfLink(selfLink);
+    bucketPb.setVersioning(new Versioning().setEnabled(versioningEnabled));
+    Website website = new Website();
     website.setMainPageSuffix(indexPage);
     website.setNotFoundPage(notFoundPage);
-    bucket.setWebsite(website);
-    Bucket.Lifecycle lifecycle = new Bucket.Lifecycle();
-    lifecycle.setRule(Lists.transform(deleteRules, new Function() {
+    bucketPb.setWebsite(website);
+    Lifecycle lifecycle = new Lifecycle();
+    lifecycle.setRule(transform(deleteRules, new Function() {
       @Override public Rule apply(DeleteRule deleteRule) {
         return deleteRule.toPb();
       }
     }));
-    return bucket;
+    return bucketPb;
+  }
+
+  static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) {
+    Builder builder = new Builder(bucketPb.getId(), bucketPb.getName())
+        .createTime(bucketPb.getTimeCreated().getValue())
+        .etag(bucketPb.getEtag())
+        .metageneration(bucketPb.getMetageneration())
+        .location(Location.of(bucketPb.getLocation()))
+        .storageClass(StorageClass.of(bucketPb.getStorageClass()))
+        .cors(transform(bucketPb.getCors(),
+            new Function() {
+              @Override public Cors apply(com.google.api.services.storage.model.Bucket.Cors cors) {
+                return Cors.fromPb(cors);
+              }
+            }))
+        .acl(transform(bucketPb.getAcl(),
+            new Function() {
+              @Override public Acl apply(BucketAccessControl bucketAccessControl) {
+                return Acl.fromPb(bucketAccessControl);
+              }
+            }))
+        .defaultAcl(transform(bucketPb.getDefaultObjectAcl(),
+            new Function() {
+              @Override public Acl apply(ObjectAccessControl objectAccessControl) {
+                return Acl.fromPb(objectAccessControl);
+              }
+            }))
+        .owner(Entity.fromPb(bucketPb.getOwner().getEntity()))
+        .selfLink(bucketPb.getSelfLink());
+    Versioning versioning = bucketPb.getVersioning();
+    if (versioning != null) {
+      builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE));
+    }
+    Website website = bucketPb.getWebsite();
+    if (website != null) {
+      builder.indexPage(website.getMainPageSuffix());
+      builder.notFoundPage(website.getNotFoundPage());
+    }
+    Lifecycle lifecycle = bucketPb.getLifecycle();
+    if (lifecycle != null) {
+      builder.deleteRules(transform(lifecycle.getRule(),
+          new Function() {
+            @Override public DeleteRule apply(Rule rule) {
+              return DeleteRule.fromPb(rule);
+            }
+          }));
+    }
+    return builder.build();
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
index 8cc86d2c3e00..76d190d81a44 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
+++ b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
@@ -32,4 +32,6 @@ public interface ObjectWriteChannel extends WritableByteChannel, Serializable, C
   String bucket();
 
   String object();
+
+  // todo: return ObjectInfo? upon close?
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index f88f68cb331f..6ad5397384b6 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -20,23 +20,26 @@
 
 public interface StorageService extends Service {
 
-  BucketInfo get(String bucket);
+  //todo: implement add/delete bucket
+  //todo: consider what to do with predefinedAcl
 
-  ObjectInfo get(String bucket, String object);
+  Bucket get(String bucket);
 
-  Iterable list(ListOptions settings);
+  Blob get(String bucket, String blob);
 
-  void update(BucketInfo bucketInfo);
+  Iterable list(ListOptions settings);
 
-  void update(ObjectInfo objectInfo);
+  Bucket update(Bucket bucket);
 
-  void delete(String bucket, String object);
+  Blob update(Blob blob);
 
-  void compose(String bucket, Iterable sourceObjects, String destObject);
+  void delete(String bucket, String blob);
 
-  void copy(String fromBucket, String fromObject, String destBucket, String destObject);
+  Blob compose(String bucket, Iterable srcBlobs, String destBlob);
 
-  ObjectReadChannel newReader(String bucket, String ObjectName);
+  Blob copy(String srcBucket, String srcBlob, String destBucket, String destBlob);
 
-  ObjectWriteChannel newWriter(String ObjectName);
+  ObjectReadChannel newReader(String bucket, String blob);
+
+  ObjectWriteChannel newWriter(Blob blob);
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 96be7f634056..64ca284945da 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,9 +16,6 @@
 
 package com.google.gcloud.storage;
 
-import com.google.api.services.storage.model.Bucket;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.spi.StorageRpc;
 
@@ -33,26 +30,63 @@ final class StorageServiceImpl extends BaseService implem
     storageRpc = options.storageRpc();
   }
 
+
   @Override
-  public Iterable listBuckets() {
+  public Bucket get(String bucket) {
     try {
-      return Iterables.transform(storageRpc.buckets(),
-          new Function() {
-            @Override public Bucket apply(com.google.api.services.storage.model.Bucket model) {
-              return new BucketImpl(StorageServiceImpl.this, model);
-            }
-          });
+      return Bucket.fromPb(storageRpc.get(bucket));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
   }
 
   @Override
-  public BucketInfo getBucket(String bucket) {
+  public Blob get(String bucket, String object) {
     try {
-      return new BucketImpl(this, storageRpc.bucket(bucket));
+      return Blob.fromPb(storageRpc.get(bucket, object));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
   }
+
+  @Override
+  public Iterable list(ListOptions settings) {
+    return null;
+  }
+
+  @Override
+  public Bucket update(Bucket bucket) {
+    return null;
+  }
+
+  @Override
+  public Blob update(Blob blob) {
+    return null;
+  }
+
+  @Override
+  public void delete(String bucket, String object) {
+
+  }
+
+  @Override
+  public Blob compose(String bucket, Iterable sourceObjects, String destObject) {
+    return null;
+  }
+
+  @Override
+  public Blob copy(String fromBucket, String fromObject, String destBucket,
+      String destObject) {
+    return null;
+  }
+
+  @Override
+  public ObjectReadChannel newReader(String bucket, String ObjectName) {
+    return null;
+  }
+
+  @Override
+  public ObjectWriteChannel newWriter(Blob blob) {
+    return null;
+  }
 }

From 5871367832a6d24035b26eb3e585a549a2edfefb Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 14 Apr 2015 14:54:27 -0700
Subject: [PATCH 165/732] work in progress

---
 .../google/gcloud/spi/DefaultStorageRpc.java  |   2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../java/com/google/gcloud/storage/Blob.java  |  55 ++++++---
 ...tReadChannel.java => BlobReadChannel.java} |  14 ++-
 .../com/google/gcloud/storage/Bucket.java     |  60 ++++++----
 .../java/com/google/gcloud/storage/Cors.java  |  12 ++
 .../google/gcloud/storage/ListOptions.java    |   2 +-
 .../gcloud/storage/ObjectWriteChannel.java    |   5 -
 .../google/gcloud/storage/StorageService.java |  34 ++++--
 .../gcloud/storage/StorageServiceImpl.java    | 110 +++++++++++++++---
 10 files changed, 216 insertions(+), 80 deletions(-)
 rename src/main/java/com/google/gcloud/storage/{ObjectReadChannel.java => BlobReadChannel.java} (82%)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 44dc5df4e86a..b61f96d23c06 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public List objects(String bucket, String prefix, String delimiter)
      throws IOException {
    return null;
  }

  @Override
  public Bucket get(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }

  @Override
  public StorageObject get(String bucket, String object) throws IOException {
    return storage.objects().get(bucket, object).execute();
  }

  @Override
  public Bucket patch(Bucket bucket) throws IOException {
    return storage.buckets().patch(bucket.getName(), bucket).execute();
  }

  @Override
  public StorageObject patch(StorageObject storageObject) throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
  }

  @Override
  public void delete(String bucket, String object) throws IOException {
    storage.objects().delete(bucket, object).execute();
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();

    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public List objects(String bucket, String prefix, String delimiter)
      throws IOException {
    return null;
  }

  @Override
  public Bucket get(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }

  @Override
  public StorageObject get(String bucket, String object) throws IOException {
    return storage.objects().get(bucket, object).execute();
  }

  @Override
  public Bucket patch(Bucket bucket) throws IOException {
    return storage.buckets().patch(bucket.getName(), bucket).execute();
  }

  @Override
  public StorageObject patch(StorageObject storageObject) throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
  }

  @Override
  public void delete(String bucket, String object) throws IOException {
    storage.objects().delete(bucket, object).execute();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 9c650f5113fe..d74e8aeddc43 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;

import java.io.IOException;
import java.util.List;

public interface StorageRpc {
  List buckets() throws IOException;
  List objects(String bucket, String prefix, String delimiter) throws IOException;
  Bucket get(String bucket) throws IOException;
  StorageObject get(String bucket, String object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  //void delete(String bucket) throws  IOException;
  void delete(String bucket, String object) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.ObjectWriteChannel;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;

public interface StorageRpc {

  Bucket create(Bucket bucket) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content) throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(Bucket bucket) throws IOException;
  StorageObject get(StorageObject object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  void delete(Bucket bucket) throws  IOException;
  void delete(StorageObject object) throws IOException;
  StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException;
  StorageObject copy(StorageObject from, StorageObject to) throws IOException;
  BlobReadChannel reader(StorageObject from) throws IOException;
  ObjectWriteChannel writer(StorageObject to) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index e5fb3a76a6e2..d55673bcc168 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -16,6 +16,8 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.api.services.storage.model.StorageObject;
@@ -34,6 +36,19 @@ public class Blob implements Serializable {
 
   private static final long serialVersionUID = 2228487739943277159L;
 
+  static final Function FROM_PB_FUNCTION =
+      new Function() {
+        @Override public Blob apply(StorageObject pb) {
+          return Blob.fromPb(pb);
+        }
+      };
+
+  static final Function TO_PB_FUNCTION = new Function() {
+        @Override public StorageObject apply(Blob blob) {
+          return blob.toPb();
+        }
+      };
+
   private final String bucket;
   private final String id;
   private final String name;
@@ -86,7 +101,7 @@ private Builder() {
     }
 
     public Builder bucket(String bucket) {
-      this.bucket = bucket;
+      this.bucket = checkNotNull(bucket);
       return this;
     }
 
@@ -96,7 +111,7 @@ Builder id(String id) {
     }
 
     public Builder name(String name) {
-      this.name = name;
+      this.name = checkNotNull(name);
       return this;
     }
 
@@ -201,9 +216,9 @@ public Blob build() {
   }
 
   private Blob(Builder builder) {
-    bucket = builder.bucket;
+    bucket = checkNotNull(builder.bucket);
+    name = checkNotNull(builder.name);
     id = builder.id;
-    name = builder.name;
     cacheControl = builder.cacheControl;
     contentEncoding = builder.contentEncoding;
     contentType = builder.contentType;
@@ -314,33 +329,37 @@ public long updateTime() {
   }
 
   public Builder toBuilder() {
-    return builder()
-        .acl(acl)
+    return new Builder()
         .bucket(bucket)
+        .name(name)
+        .id(id)
+        .generation(generation)
         .cacheControl(cacheControl)
         .contentEncoding(contentEncoding)
-        .crc32c(crc32c)
         .contentType(contentType)
-        .deleteTime(deleteTime)
-        .generation(generation)
+        .contentDisposition(contentDisposition)
+        .contentLanguage(contentLanguage)
+        .componentCount(componentCount)
+        .crc32c(crc32c)
         .md5(md5)
+        .deleteTime(deleteTime)
+        .updateTime(updateTime)
         .mediaLink(mediaLink)
         .metadata(metadata)
         .metageneration(metageneration)
-        .name(name)
+        .acl(acl)
         .owner(owner)
-        .updateTime(updateTime)
         .size(size)
-        .contentDisposition(contentDisposition)
-        .componentCount(componentCount)
-        .contentLanguage(contentLanguage)
         .etag(etag)
-        .id(id)
         .selfLink(selfLink);
   }
 
-  public static Builder builder() {
-    return new Builder();
+  public static Builder builder(Bucket bucket, String name) {
+    return builder(bucket.name(), name);
+  }
+
+  public static Builder builder(String bucket, String name) {
+    return new Builder().bucket(bucket).name(name);
   }
 
   StorageObject toPb() {
@@ -376,7 +395,7 @@ StorageObject toPb() {
   }
 
   static Blob fromPb(StorageObject storageObject) {
-    return builder()
+    return new Builder()
         .acl(Lists.transform(storageObject.getAcl(),
             new Function() {
               @Override public Acl apply(ObjectAccessControl objectAccessControl) {
diff --git a/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
similarity index 82%
rename from src/main/java/com/google/gcloud/storage/ObjectReadChannel.java
rename to src/main/java/com/google/gcloud/storage/BlobReadChannel.java
index 0dbb320685f1..3757dec04e0b 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectReadChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
@@ -27,13 +27,15 @@
  *
  *  This class is @{link Serializable}, which allows incremental reads.
  */
-public interface ObjectReadChannel extends ReadableByteChannel, Serializable, Closeable {
+public interface BlobReadChannel extends ReadableByteChannel, Serializable, Closeable {
 
-  String bucket();
-
-  String object();
-
-  int size();
+  /**
+   * Overridden to remove IOException.
+   *
+   * @see java.nio.channels.Channel#close()
+   */
+  @Override
+  void close();
 
   void seek(int position);
 }
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 7d3944c0d9c0..4771506a2a35 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -58,6 +58,20 @@ public final class Bucket implements Serializable {
   private final StorageClass storageClass;
 
 
+  static final Function FROM_PB_FUNCTION =
+      new Function() {
+        @Override public Bucket apply(com.google.api.services.storage.model.Bucket pb) {
+          return Bucket.fromPb(pb);
+        }
+      };
+
+  static final Function TO_PB_FUNCTION =
+      new Function() {
+        @Override public com.google.api.services.storage.model.Bucket apply(Bucket bucket) {
+          return bucket.toPb();
+        }
+      };
+
   public static abstract class DeleteRule implements Serializable {
 
     private static final long serialVersionUID = 3137971668395933033L;
@@ -87,7 +101,7 @@ Rule toPb() {
 
     abstract void populateCondition(Rule.Condition condition);
 
-    static DeleteRule fromPb(Rule rule) {
+    private static DeleteRule fromPb(Rule rule) {
       if (rule.getAction() != null && SUPPORTED_ACTION.endsWith(rule.getAction().getType())) {
         Rule.Condition condition = rule.getCondition();
         Integer age = condition.getAge();
@@ -314,8 +328,8 @@ public String value() {
 
   public final static class Builder {
 
-    private final String id;
-    private final String name;
+    private String id;
+    private String name;
     private Acl.Entity owner;
     private String selfLink;
     private boolean versioningEnabled;
@@ -331,9 +345,17 @@ public final static class Builder {
     private Iterable acl = ImmutableList.of();
     private Iterable defaultAcl = ImmutableList.of();
 
-    Builder(String id, String name) {
+    private Builder() {
+    }
+
+    public Builder name(String name) {
+      this.name = checkNotNull(name);
+      return this;
+    }
+
+    Builder id(String id) {
       this.id = id;
-      this.name = name;
+      return this;
     }
 
     Builder owner(Acl.Entity owner) {
@@ -413,7 +435,7 @@ public Bucket build() {
 
   private Bucket(Builder builder) {
     id = builder.id;
-    name = builder.name;
+    name = checkNotNull(builder.name);
     etag = builder.etag;
     createTime = MoreObjects.firstNonNull(builder.createTime, 0L);
     metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L);
@@ -495,7 +517,9 @@ public List defaultAcl() {
   }
 
   public Builder toBuilder() {
-    return new Builder(id, name)
+    return new Builder()
+        .name(name)
+        .id(id)
         .createTime(createTime)
         .etag(etag)
         .metageneration(metageneration)
@@ -512,6 +536,10 @@ public Builder toBuilder() {
         .deleteRules(deleteRules);
   }
 
+  public static Builder builder(String name) {
+    return new Builder().name(name);
+  }
+
   com.google.api.services.storage.model.Bucket toPb() {
     com.google.api.services.storage.model.Bucket bucketPb =
         new com.google.api.services.storage.model.Bucket();
@@ -530,12 +558,7 @@ com.google.api.services.storage.model.Bucket toPb() {
     if (storageClass != null) {
       bucketPb.setStorageClass(storageClass.value());
     }
-    bucketPb.setCors(
-        transform(cors, new Function() {
-          @Override public com.google.api.services.storage.model.Bucket.Cors apply(Cors cors) {
-            return cors.toPb();
-          }
-        }));
+    bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION));
     bucketPb.setAcl(transform(acl, new Function() {
       @Override public BucketAccessControl apply(Acl acl) {
         return acl.toBucketPb();
@@ -564,18 +587,15 @@ com.google.api.services.storage.model.Bucket toPb() {
   }
 
   static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) {
-    Builder builder = new Builder(bucketPb.getId(), bucketPb.getName())
+    Builder builder = new Builder()
+        .name(bucketPb.getName())
+        .id(bucketPb.getId())
         .createTime(bucketPb.getTimeCreated().getValue())
         .etag(bucketPb.getEtag())
         .metageneration(bucketPb.getMetageneration())
         .location(Location.of(bucketPb.getLocation()))
         .storageClass(StorageClass.of(bucketPb.getStorageClass()))
-        .cors(transform(bucketPb.getCors(),
-            new Function() {
-              @Override public Cors apply(com.google.api.services.storage.model.Bucket.Cors cors) {
-                return Cors.fromPb(cors);
-              }
-            }))
+        .cors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION))
         .acl(transform(bucketPb.getAcl(),
             new Function() {
               @Override public Acl apply(BucketAccessControl bucketAccessControl) {
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index a58331d5063a..87d33348983d 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -34,6 +34,18 @@ public final class Cors implements Serializable {
 
   private static final long serialVersionUID = -8637770919343335655L;
 
+  static final Function FROM_PB_FUNCTION = new Function() {
+    @Override public Cors apply(Bucket.Cors pb) {
+      return Cors.fromPb(pb);
+    }
+  };
+
+  static final Function TO_PB_FUNCTION = new Function() {
+    @Override public Bucket.Cors apply(Cors cors) {
+      return cors.toPb();
+    }
+  };
+
   private final Integer maxAgeSeconds;
   private final ImmutableList methods;
   private final ImmutableList origins;
diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java
index b1b5d96f27da..06637ed6aa3b 100644
--- a/src/main/java/com/google/gcloud/storage/ListOptions.java
+++ b/src/main/java/com/google/gcloud/storage/ListOptions.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;

    public Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    public Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
index 76d190d81a44..518fb049718d 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
+++ b/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
@@ -29,9 +29,4 @@
  */
 public interface ObjectWriteChannel extends WritableByteChannel, Serializable, Closeable {
 
-  String bucket();
-
-  String object();
-
-  // todo: return ObjectInfo? upon close?
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 6ad5397384b6..6dfc8c48ef90 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -18,28 +18,42 @@
 
 import com.google.gcloud.Service;
 
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
 public interface StorageService extends Service {
 
-  //todo: implement add/delete bucket
-  //todo: consider what to do with predefinedAcl
+  // todo: consider what to do with predefinedAcl
+  // todo: consider supplying create/update/delete options (varargs) for
+  // ifGenerationMatch, ifGenerationNotMatch, ifMetagenerationMatch,
+  // ifMetagenerationNotMatch, ifSourceGenerationMatch, ifSourceGenerationNotMatch
+  // ifSourceMetagenerationMatch and ifSourceMetagenerationNotMatch
+
+  Bucket create(Bucket bucket);
+
+  Blob create(Blob blob, ByteBuffer content);
 
-  Bucket get(String bucket);
+  Bucket get(Bucket bucket);
 
-  Blob get(String bucket, String blob);
+  Blob get(Blob blob);
 
-  Iterable list(ListOptions settings);
+  Iterator list();
+
+  Iterator list(Bucket bucket, ListOptions settings);
 
   Bucket update(Bucket bucket);
 
   Blob update(Blob blob);
 
-  void delete(String bucket, String blob);
+  void delete(Bucket bucket);
+
+  void delete(Blob blob);
 
-  Blob compose(String bucket, Iterable srcBlobs, String destBlob);
+  Blob compose(Bucket bucket, Iterable src, Blob dest);
 
-  Blob copy(String srcBucket, String srcBlob, String destBucket, String destBlob);
+  Blob copy(Blob src, Blob dest);
 
-  ObjectReadChannel newReader(String bucket, String blob);
+  BlobReadChannel readFrom(Blob blob); // todo: consider returning Blob
 
-  ObjectWriteChannel newWriter(Blob blob);
+  ObjectWriteChannel writeTo(Blob blob); // todo: consider returning Blob
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 64ca284945da..96b08398645c 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,10 +16,13 @@
 
 package com.google.gcloud.storage;
 
+import com.google.common.collect.Iterators;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.spi.StorageRpc;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
 
 final class StorageServiceImpl extends BaseService implements StorageService {
 
@@ -32,61 +35,132 @@ final class StorageServiceImpl extends BaseService implem
 
 
   @Override
-  public Bucket get(String bucket) {
+  public Bucket create(Bucket bucket) {
     try {
-      return Bucket.fromPb(storageRpc.get(bucket));
+      return Bucket.fromPb(storageRpc.create(bucket.toPb()));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
   }
 
   @Override
-  public Blob get(String bucket, String object) {
+  public Blob create(Blob blob, ByteBuffer content) {
     try {
-      return Blob.fromPb(storageRpc.get(bucket, object));
+      return Blob.fromPb(storageRpc.create(blob.toPb(), content));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
   }
 
   @Override
-  public Iterable list(ListOptions settings) {
-    return null;
+  public Bucket get(Bucket bucket) {
+    try {
+      return Bucket.fromPb(storageRpc.get(bucket.toPb()));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
+  }
+
+  @Override
+  public Blob get(Blob blob) {
+    try {
+      return Blob.fromPb(storageRpc.get(blob.toPb()));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
+  }
+
+  @Override
+  public Iterator list() {
+    try {
+      return Iterators.transform(storageRpc.list(), Bucket.FROM_PB_FUNCTION);
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
+  }
+
+  @Override
+  public Iterator list(Bucket bucket, ListOptions settings) {
+    try {
+      String delimiter = settings.recursive() ? options().pathDelimiter() : null;
+      return Iterators.transform(
+          storageRpc.list(bucket.name(), settings.prefix(), delimiter, settings.cursor(),
+              settings.includeOlderVersions(), settings.maxResults()),
+          Blob.FROM_PB_FUNCTION);
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
   public Bucket update(Bucket bucket) {
-    return null;
+    try {
+      return Bucket.fromPb(storageRpc.patch(bucket.toPb()));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
   public Blob update(Blob blob) {
-    return null;
+    try {
+      return Blob.fromPb(storageRpc.patch(blob.toPb()));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
-  public void delete(String bucket, String object) {
+  public void delete(Bucket bucket) {
+    try {
+      storageRpc.delete(bucket.toPb());
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
+  }
 
+  @Override
+  public void delete(Blob blob) {
+    try {
+      storageRpc.delete(blob.toPb());
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
-  public Blob compose(String bucket, Iterable sourceObjects, String destObject) {
-    return null;
+  public Blob compose(Bucket bucket, Iterable src, Blob dest) {
+    try {
+      return Blob.fromPb(storageRpc.compose(bucket.name(), src, dest.toPb()));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
-  public Blob copy(String fromBucket, String fromObject, String destBucket,
-      String destObject) {
-    return null;
+  public Blob copy(Blob src, Blob dest) {
+    try {
+      return Blob.fromPb(storageRpc.copy(src.toPb(), dest.toPb()));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
-  public ObjectReadChannel newReader(String bucket, String ObjectName) {
-    return null;
+  public BlobReadChannel readFrom(Blob blob) {
+    try {
+      return storageRpc.reader(blob.toPb());
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
-  public ObjectWriteChannel newWriter(Blob blob) {
-    return null;
+  public ObjectWriteChannel writeTo(Blob blob) {
+    try {
+      return storageRpc.writer(blob.toPb());
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 }

From 82c1ffb015ae331f99c106e74bdc666f7f23e7cf Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 14 Apr 2015 14:59:09 -0700
Subject: [PATCH 166/732] work in progress

---
 src/main/java/com/google/gcloud/spi/StorageRpc.java             | 2 +-
 .../storage/{ObjectWriteChannel.java => BlobWriteChannel.java}  | 2 +-
 src/main/java/com/google/gcloud/storage/StorageService.java     | 2 +-
 src/main/java/com/google/gcloud/storage/StorageServiceImpl.java | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)
 rename src/main/java/com/google/gcloud/storage/{ObjectWriteChannel.java => BlobWriteChannel.java} (92%)

diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index d74e8aeddc43..3e272eeb4606 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.ObjectWriteChannel;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;

public interface StorageRpc {

  Bucket create(Bucket bucket) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content) throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(Bucket bucket) throws IOException;
  StorageObject get(StorageObject object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  void delete(Bucket bucket) throws  IOException;
  void delete(StorageObject object) throws IOException;
  StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException;
  StorageObject copy(StorageObject from, StorageObject to) throws IOException;
  BlobReadChannel reader(StorageObject from) throws IOException;
  ObjectWriteChannel writer(StorageObject to) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;

public interface StorageRpc {

  Bucket create(Bucket bucket) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content) throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(Bucket bucket) throws IOException;
  StorageObject get(StorageObject object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  void delete(Bucket bucket) throws  IOException;
  void delete(StorageObject object) throws IOException;
  StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException;
  StorageObject copy(StorageObject from, StorageObject to) throws IOException;
  BlobReadChannel reader(StorageObject from) throws IOException;
  BlobWriteChannel writer(StorageObject to) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
similarity index 92%
rename from src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
rename to src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
index 518fb049718d..a83ed661baca 100644
--- a/src/main/java/com/google/gcloud/storage/ObjectWriteChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
@@ -27,6 +27,6 @@
  * Written data will only be visible after calling {@link #close()}.
  * This class is serializable, to allow incremental writes.
  */
-public interface ObjectWriteChannel extends WritableByteChannel, Serializable, Closeable {
+public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable {
 
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 6dfc8c48ef90..2828f612b4ca 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -55,5 +55,5 @@ public interface StorageService extends Service {
 
   BlobReadChannel readFrom(Blob blob); // todo: consider returning Blob
 
-  ObjectWriteChannel writeTo(Blob blob); // todo: consider returning Blob
+  BlobWriteChannel writeTo(Blob blob); // todo: consider returning Blob
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 96b08398645c..eb9f78684efd 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -156,7 +156,7 @@ public BlobReadChannel readFrom(Blob blob) {
   }
 
   @Override
-  public ObjectWriteChannel writeTo(Blob blob) {
+  public BlobWriteChannel writeTo(Blob blob) {
     try {
       return storageRpc.writer(blob.toPb());
     } catch (IOException ex) {

From 02a8550fe942da228cf64a03437ad0abef1c6558 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 14 Apr 2015 15:57:24 -0700
Subject: [PATCH 167/732] work in progress

---
 src/main/java/com/google/gcloud/spi/StorageRpc.java  |  2 +-
 .../com/google/gcloud/storage/StorageService.java    |  9 ++++++---
 .../google/gcloud/storage/StorageServiceImpl.java    | 12 ++++++------
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 3e272eeb4606..2e851cdba981 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;

public interface StorageRpc {

  Bucket create(Bucket bucket) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content) throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(Bucket bucket) throws IOException;
  StorageObject get(StorageObject object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  void delete(Bucket bucket) throws  IOException;
  void delete(StorageObject object) throws IOException;
  StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException;
  StorageObject copy(StorageObject from, StorageObject to) throws IOException;
  BlobReadChannel reader(StorageObject from) throws IOException;
  BlobWriteChannel writer(StorageObject to) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;

public interface StorageRpc {

  Bucket create(Bucket bucket) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content) throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(String bucket) throws IOException;
  StorageObject get(String bucket, String object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  void delete(Bucket bucket) throws  IOException;
  void delete(StorageObject object) throws IOException;
  StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException;
  StorageObject copy(StorageObject from, StorageObject to) throws IOException;
  BlobReadChannel reader(StorageObject from) throws IOException;
  BlobWriteChannel writer(StorageObject to) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 2828f612b4ca..ff57f3fa2906 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -24,22 +24,25 @@
 public interface StorageService extends Service {
 
   // todo: consider what to do with predefinedAcl
+
   // todo: consider supplying create/update/delete options (varargs) for
   // ifGenerationMatch, ifGenerationNotMatch, ifMetagenerationMatch,
   // ifMetagenerationNotMatch, ifSourceGenerationMatch, ifSourceGenerationNotMatch
   // ifSourceMetagenerationMatch and ifSourceMetagenerationNotMatch
 
+  // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs
+
   Bucket create(Bucket bucket);
 
   Blob create(Blob blob, ByteBuffer content);
 
-  Bucket get(Bucket bucket);
+  Bucket get(String bucket);
 
-  Blob get(Blob blob);
+  Blob get(String bucket, String blob);
 
   Iterator list();
 
-  Iterator list(Bucket bucket, ListOptions settings);
+  Iterator list(String bucket, ListOptions settings);
 
   Bucket update(Bucket bucket);
 
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index eb9f78684efd..6df6800d31d5 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -53,18 +53,18 @@ public Blob create(Blob blob, ByteBuffer content) {
   }
 
   @Override
-  public Bucket get(Bucket bucket) {
+  public Bucket get(String bucket) {
     try {
-      return Bucket.fromPb(storageRpc.get(bucket.toPb()));
+      return Bucket.fromPb(storageRpc.get(bucket));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
   }
 
   @Override
-  public Blob get(Blob blob) {
+  public Blob get(String bucket, String blob) {
     try {
-      return Blob.fromPb(storageRpc.get(blob.toPb()));
+      return Blob.fromPb(storageRpc.get(bucket, blob));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -80,11 +80,11 @@ public Iterator list() {
   }
 
   @Override
-  public Iterator list(Bucket bucket, ListOptions settings) {
+  public Iterator list(String bucket, ListOptions settings) {
     try {
       String delimiter = settings.recursive() ? options().pathDelimiter() : null;
       return Iterators.transform(
-          storageRpc.list(bucket.name(), settings.prefix(), delimiter, settings.cursor(),
+          storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(),
               settings.includeOlderVersions(), settings.maxResults()),
           Blob.FROM_PB_FUNCTION);
     } catch (IOException ex) {

From 4612cf96638fe0df24c7f2878c939811256769d4 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 15 Apr 2015 17:33:40 -0700
Subject: [PATCH 168/732] work in progress

---
 .../gcloud/examples/StorageExample.java       |   5 +-
 .../google/gcloud/storage/StorageService.java | 334 +++++++++++++++++-
 .../gcloud/storage/StorageServiceImpl.java    |  49 ++-
 3 files changed, 349 insertions(+), 39 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index ebfbeef3545f..f670cc7ac1db 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.examples;
 
+import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.StorageService;
 import com.google.gcloud.storage.StorageServiceFactory;
@@ -61,12 +62,11 @@ protected String fullPath(String folder, String file) {
     }
   }
 
-
   private static class DeleteAction extends StorageAction {
     @Override
     public void run(StorageService storage, Bucket bucket, String folder, String... args) {
       for (String file : args) {
-        storage.delete(bucket.name(), fullPath(folder, file));
+        storage.delete(Blob.builder(bucket, fullPath(folder, file)).build());
       }
     }
 
@@ -78,6 +78,7 @@ public String getRequiredParams() {
 
   static {
     ACTIONS.put("delete", new DeleteAction());
+    // todo: implement list, get and put
   }
 
   public static void main(String... args) {
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index ff57f3fa2906..5ce0c3d701bc 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -16,47 +16,345 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
 import com.google.gcloud.Service;
 
+import java.io.Serializable;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 public interface StorageService extends Service {
 
-  // todo: consider what to do with predefinedAcl
+  // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs
 
-  // todo: consider supplying create/update/delete options (varargs) for
-  // ifGenerationMatch, ifGenerationNotMatch, ifMetagenerationMatch,
-  // ifMetagenerationNotMatch, ifSourceGenerationMatch, ifSourceGenerationNotMatch
-  // ifSourceMetagenerationMatch and ifSourceMetagenerationNotMatch
+  enum PredefinedAcl {
+    AUTHENTICATED_READ("authenticatedRead"),
+    ALL_AUTHENTICATED_USERS("allAuthenticatedUsers"),
+    PRIVATE("private"),
+    PROJECT_PRIVATE("projectPrivate"),
+    PUBLIC_READ("publicRead"),
+    PUBLIC_READ_WRITE("publicReadWrite"),
+    BUCKET_OWNER_READ("bucketOwnerRead"),
+    BUCKET_OWNER_FULL_CONTROL("bucketOwnerFullControl");
 
-  // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs
+    private final String entry;
+
+    PredefinedAcl(String entry) {
+      this.entry = entry;
+    }
+
+    String entry() {
+      return entry;
+    }
+  }
+
+  class Option implements Serializable {
+
+    private static final long serialVersionUID = -73199088766477208L;
+
+    private final String name;
+    private final Object value;
+
+    Option(String name, Object value) {
+      this.name = checkNotNull(name);
+      this.value = value;
+    }
+
+    String name() {
+      return name;
+    }
+
+    Object value() {
+      return value;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!(obj instanceof Option)) {
+        return false;
+      }
+      Option other = (Option) obj;
+      return Objects.equals(name, other.name)
+          && Objects.equals(value, other.value);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(name, value);
+    }
+  }
+
+  class BucketTargetOption extends Option {
+
+    private static final long serialVersionUID = -5880204616982900975L;
+
+    private BucketTargetOption(String name, Object value) {
+      super(name, value);
+    }
+
+    public static BucketTargetOption predefinedAcl(PredefinedAcl acl) {
+      return new BucketTargetOption("predefinedAcl", acl.entry());
+    }
+
+    public static BucketTargetOption predefinedDefaultObjectAcl(PredefinedAcl acl) {
+      return new BucketTargetOption("predefinedDefaultObjectAcl", acl.entry());
+    }
+
+    public static BucketTargetOption metagenerationMatch(boolean match) {
+      return new BucketTargetOption("ifMetagenerationMatch", match);
+    }
+  }
+
+  class BucketSourceOption extends Option {
+
+    private static final long serialVersionUID = 5185657617120212117L;
+
+    private BucketSourceOption(String name, Object value) {
+      super(name, value);
+    }
+
+    public static BucketSourceOption metagenerationMatch(boolean match) {
+      return new BucketSourceOption("ifMetagenerationMatch", match);
+    }
+  }
+
+  class BlobTargetOption extends Option {
+
+    private static final long serialVersionUID = 214616862061934846L;
+
+    private BlobTargetOption(String name, Object value) {
+      super(name, value);
+    }
+
+    public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
+      return new BlobTargetOption("predefinedAcl", acl.entry());
+    }
+
+    public static BlobTargetOption generationMath(boolean match) {
+      return new BlobTargetOption("ifMetagenerationMatch", match);
+    }
+
+    public static BlobTargetOption metagenerationMatch(boolean match) {
+      return new BlobTargetOption("ifMetagenerationMatch", match);
+    }
+  }
+
+  class BlobSourceOption extends Option {
+
+    private static final long serialVersionUID = -3712768261070182991L;
+
+    private BlobSourceOption(String name, Object value) {
+      super(name, value);
+    }
+
+    public static BlobSourceOption generationMath(boolean match) {
+      return new BlobSourceOption("ifGenerationMatch", match);
+    }
+
+    public static BlobSourceOption metagenerationMatch(boolean match) {
+      return new BlobSourceOption("ifMetagenerationMatch", match);
+    }
+  }
+
+  class ComposeRequest implements Serializable {
+
+    private static final long serialVersionUID = -7385681353748590911L;
+
+    private final String sourceBucket;
+    private final List sourceBlobs;
+    private final Blob target;
+    private final List targetOptions;
+
+    private static class SourceBlob implements Serializable {
+
+      private static final long serialVersionUID = 4094962795951990439L;
+
+      final String blob;
+      final String generation;
+
+      SourceBlob(String blob) {
+        this(blob, null);
+      }
+
+      SourceBlob(String blob, String generation) {
+        this.blob = blob;
+        this.generation = generation;
+      }
+    }
+
+    public static class Builder {
+
+      private String bucket;
+      private List sourceBlobs = new LinkedList<>();
+      private Blob target;
+      private Set targetOptions = new LinkedHashSet<>();
+
+      public Builder sourceBucket(String bucket) {
+        this.bucket = bucket;
+        return this;
+      }
+
+      public Builder addSource(Iterable blobs) {
+        for (String blob : blobs) {
+          sourceBlobs.add(new SourceBlob(blob));
+        }
+        return this;
+      }
+
+      public Builder addSource(String... blobs) {
+        return addSource(Arrays.asList(blobs));
+      }
+
+      public Builder addSource(String blob, String generation) {
+        sourceBlobs.add(new SourceBlob(blob, generation));
+        return this;
+      }
+
+      public Builder target(Blob target) {
+        this.target = target;
+        return this;
+      }
+
+      public Builder targetOptions(BlobTargetOption... options) {
+        Collections.addAll(targetOptions, options);
+        return this;
+      }
+
+      public ComposeRequest build() {
+        return new ComposeRequest(this);
+      }
+    }
+
+    private ComposeRequest(Builder builder) {
+      sourceBucket = checkNotNull(builder.bucket);
+      sourceBlobs = ImmutableList.copyOf(builder.sourceBlobs);
+      target = checkNotNull(builder.target);
+      targetOptions = ImmutableList.copyOf(builder.targetOptions);
+    }
+
+    String sourceBucket() {
+      return sourceBucket;
+    }
+
+    List sourceBlobs() {
+      return sourceBlobs;
+    }
+
+    Blob target() {
+      return target;
+    }
+
+    List targetOptions() {
+      return targetOptions;
+    }
+
+    public static Builder builder() {
+      return new Builder();
+    }
+  }
+
+  class CopyRequest implements Serializable {
+
+    private static final long serialVersionUID = -2606508373751748775L;
+
+    private final Blob source;
+    private final List sourceOptions;
+    private final Blob target;
+    private final List targetOptions;
+
+    public static class Builder {
+
+      private Blob source;
+      private Set sourceOptions;
+      private Blob target;
+      private Set targetOptions;
+
+      public void source(Blob source) {
+        this.source = source;
+      }
+
+      public Builder sourceOptions(BlobSourceOption... options) {
+        Collections.addAll(sourceOptions, options);
+        return this;
+      }
+
+      public Builder target(Blob target) {
+        this.target = target;
+        return this;
+      }
+
+      public Builder targetOptions(BlobTargetOption... options) {
+        Collections.addAll(targetOptions, options);
+        return this;
+      }
+
+      public CopyRequest build() {
+        return new CopyRequest(this);
+      }
+    }
+
+    private CopyRequest(Builder builder) {
+      source = checkNotNull(builder.source);
+      sourceOptions = ImmutableList.copyOf(builder.sourceOptions);
+      target = checkNotNull(builder.target);
+      targetOptions = ImmutableList.copyOf(builder.targetOptions);
+    }
+
+    Blob source() {
+      return source;
+    }
+
+    public List sourceOptions() {
+      return sourceOptions;
+    }
+
+    Blob target() {
+      return target;
+    }
+
+    List targetOptions() {
+      return targetOptions;
+    }
+
+    public static Builder builder() {
+      return new Builder();
+    }
+  }
 
-  Bucket create(Bucket bucket);
+  Bucket create(Bucket bucket, BucketTargetOption... option);
 
-  Blob create(Blob blob, ByteBuffer content);
+  Blob create(Blob blob, ByteBuffer content, BlobTargetOption... option);
 
-  Bucket get(String bucket);
+  Bucket get(String bucket, BucketSourceOption... options);
 
-  Blob get(String bucket, String blob);
+  Blob get(String bucket, String blob, BlobSourceOption... options);
 
   Iterator list();
 
   Iterator list(String bucket, ListOptions settings);
 
-  Bucket update(Bucket bucket);
+  Bucket update(Bucket bucket, BucketTargetOption... option);
 
-  Blob update(Blob blob);
+  Blob update(Blob blob, BlobTargetOption... option);
 
-  void delete(Bucket bucket);
+  void delete(Bucket bucket, BucketSourceOption... option);
 
-  void delete(Blob blob);
+  void delete(Blob blob, BlobSourceOption... option);
 
-  Blob compose(Bucket bucket, Iterable src, Blob dest);
+  Blob compose(ComposeRequest composeRequest);
 
-  Blob copy(Blob src, Blob dest);
+  Blob copy(CopyRequest copyRequest);
 
-  BlobReadChannel readFrom(Blob blob); // todo: consider returning Blob
+  BlobReadChannel readFrom(Blob blob, BlobSourceOption... option);
 
-  BlobWriteChannel writeTo(Blob blob); // todo: consider returning Blob
+  BlobWriteChannel writeTo(Blob blob, BlobTargetOption... option);
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 6df6800d31d5..78acb40d5b9a 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -31,11 +31,17 @@ final class StorageServiceImpl extends BaseService implem
   StorageServiceImpl(StorageServiceOptions options) {
     super(options);
     storageRpc = options.storageRpc();
-  }
 
+    // todo: like Datastore distinct exception to retriable and non-retriable
+    //       https://cloud.google.com/storage/docs/json_api/v1/status-codes
+    // todo: Use retry helper on retriable failures
+
+    // todo: consider options
+    // todo: replace nulls with Value.asNull (per toPb)
+  }
 
   @Override
-  public Bucket create(Bucket bucket) {
+  public Bucket create(Bucket bucket, BucketTargetOption... options) {
     try {
       return Bucket.fromPb(storageRpc.create(bucket.toPb()));
     } catch (IOException ex) {
@@ -44,7 +50,7 @@ public Bucket create(Bucket bucket) {
   }
 
   @Override
-  public Blob create(Blob blob, ByteBuffer content) {
+  public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) {
     try {
       return Blob.fromPb(storageRpc.create(blob.toPb(), content));
     } catch (IOException ex) {
@@ -53,7 +59,7 @@ public Blob create(Blob blob, ByteBuffer content) {
   }
 
   @Override
-  public Bucket get(String bucket) {
+  public Bucket get(String bucket, BucketSourceOption... options) {
     try {
       return Bucket.fromPb(storageRpc.get(bucket));
     } catch (IOException ex) {
@@ -62,7 +68,7 @@ public Bucket get(String bucket) {
   }
 
   @Override
-  public Blob get(String bucket, String blob) {
+  public Blob get(String bucket, String blob, BlobSourceOption... options) {
     try {
       return Blob.fromPb(storageRpc.get(bucket, blob));
     } catch (IOException ex) {
@@ -93,7 +99,7 @@ public Iterator list(String bucket, ListOptions settings) {
   }
 
   @Override
-  public Bucket update(Bucket bucket) {
+  public Bucket update(Bucket bucket, BucketTargetOption... options) {
     try {
       return Bucket.fromPb(storageRpc.patch(bucket.toPb()));
     } catch (IOException ex) {
@@ -102,7 +108,7 @@ public Bucket update(Bucket bucket) {
   }
 
   @Override
-  public Blob update(Blob blob) {
+  public Blob update(Blob blob, BlobTargetOption... options) {
     try {
       return Blob.fromPb(storageRpc.patch(blob.toPb()));
     } catch (IOException ex) {
@@ -111,7 +117,7 @@ public Blob update(Blob blob) {
   }
 
   @Override
-  public void delete(Bucket bucket) {
+  public void delete(Bucket bucket, BucketSourceOption... options) {
     try {
       storageRpc.delete(bucket.toPb());
     } catch (IOException ex) {
@@ -120,7 +126,7 @@ public void delete(Bucket bucket) {
   }
 
   @Override
-  public void delete(Blob blob) {
+  public void delete(Blob blob, BlobSourceOption... options) {
     try {
       storageRpc.delete(blob.toPb());
     } catch (IOException ex) {
@@ -129,25 +135,30 @@ public void delete(Blob blob) {
   }
 
   @Override
-  public Blob compose(Bucket bucket, Iterable src, Blob dest) {
-    try {
-      return Blob.fromPb(storageRpc.compose(bucket.name(), src, dest.toPb()));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Blob compose(ComposeRequest composeRequest) {
+    // todo: implement (consider having XXXRequest/XXXResponse for all SPI/Rpc requests
+// try {
+
+//
+//      return Blob.fromPb(storageRpc.compose(composeRequest.sourceBucket(),
+//          composeRequest.sourceBlobs(), composeRequest.target()));
+//    } catch (IOException ex) {
+//      throw new StorageServiceException(ex);
+//    }
+    throw new UnsupportedOperationException("bla");
   }
 
   @Override
-  public Blob copy(Blob src, Blob dest) {
+  public Blob copy(CopyRequest copyRequest) {
     try {
-      return Blob.fromPb(storageRpc.copy(src.toPb(), dest.toPb()));
+      return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.target().toPb()));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
   }
 
   @Override
-  public BlobReadChannel readFrom(Blob blob) {
+  public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) {
     try {
       return storageRpc.reader(blob.toPb());
     } catch (IOException ex) {
@@ -156,7 +167,7 @@ public BlobReadChannel readFrom(Blob blob) {
   }
 
   @Override
-  public BlobWriteChannel writeTo(Blob blob) {
+  public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) {
     try {
       return storageRpc.writer(blob.toPb());
     } catch (IOException ex) {

From 8ebf53724ca2b42d07dd2a7aa32ded09910a89ab Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 17 Apr 2015 17:40:18 -0700
Subject: [PATCH 169/732] work in progress

---
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../java/com/google/gcloud/storage/Blob.java  | 125 +++++++------
 .../com/google/gcloud/storage/Bucket.java     | 169 ++++++++++--------
 .../com/google/gcloud/storage/Option.java     |  61 +++++++
 .../google/gcloud/storage/StorageService.java |  61 ++-----
 .../gcloud/storage/StorageServiceImpl.java    |  49 ++---
 .../gcloud/storage/StorageServiceOptions.java |  11 +-
 .../com/google/gcloud/storage/Validator.java  |  38 ++++
 8 files changed, 322 insertions(+), 194 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/Option.java
 create mode 100644 src/main/java/com/google/gcloud/storage/Validator.java

diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 2e851cdba981..74b0d55aa02e 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;

public interface StorageRpc {

  Bucket create(Bucket bucket) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content) throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(String bucket) throws IOException;
  StorageObject get(String bucket, String object) throws IOException;
  Bucket patch(Bucket bucket) throws IOException;
  StorageObject patch(StorageObject storageObject) throws IOException;
  void delete(Bucket bucket) throws  IOException;
  void delete(StorageObject object) throws IOException;
  StorageObject compose(String bucket, Iterable src, StorageObject dest) throws IOException;
  StorageObject copy(StorageObject from, StorageObject to) throws IOException;
  BlobReadChannel reader(StorageObject from) throws IOException;
  BlobWriteChannel writer(StorageObject to) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;

public interface StorageRpc {

  Bucket create(Bucket bucket, Option... options) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content, Option... options)
      throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(String bucket, Option... options) throws IOException;
  StorageObject get(String bucket, String object, Option... options) throws IOException;
  Bucket patch(Bucket bucket, Option... options) throws IOException;
  StorageObject patch(StorageObject storageObject, Option... options) throws IOException;
  void delete(Bucket bucket, Option... options) throws  IOException;
  void delete(StorageObject object, Option... options) throws IOException;
  StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws IOException;
  StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws IOException;
  BlobReadChannel reader(StorageObject from, Option... options) throws IOException;
  BlobWriteChannel writer(StorageObject to, Option... options) throws IOException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index d55673bcc168..04553a8c744e 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -54,23 +54,23 @@ public class Blob implements Serializable {
   private final String name;
   private final String selfLink;
   private final String cacheControl;
-  private final ImmutableList acl;
+  private final List acl;
   private final Acl.Entity owner;
-  private final long size;
+  private final Long size;
   private final String etag;
   private final String md5;
   private final String crc32c;
   private final String mediaLink;
-  private final ImmutableMap metadata;
-  private final long generation;
-  private final long metageneration;
-  private final long deleteTime;
-  private final long updateTime;
+  private final Map metadata;
+  private final Long generation;
+  private final Long metageneration;
+  private final Long deleteTime;
+  private final Long updateTime;
   private final String contentType;
   private final String contentEncoding;
   private final String contentDisposition;
   private final String contentLanguage;
-  private final int componentCount;
+  private final Integer componentCount;
 
   public static final class Builder {
 
@@ -80,22 +80,22 @@ public static final class Builder {
     private String contentType;
     private String contentEncoding;
     private String contentDisposition;
-    private  String contentLanguage;
-    private int componentCount;
+    private String contentLanguage;
+    private Integer componentCount;
     private String cacheControl;
     private ImmutableList acl;
     private Acl.Entity owner;
-    private long size;
+    private Long size;
     private String etag;
     private String selfLink;
     private String md5;
     private String crc32c;
     private String mediaLink;
     private ImmutableMap metadata;
-    private long generation;
-    private long metageneration;
-    private long deleteTime;
-    private long updateTime;
+    private Long generation;
+    private Long metageneration;
+    private Long deleteTime;
+    private Long updateTime;
 
     private Builder() {
     }
@@ -135,7 +135,7 @@ public Builder contentEncoding(String contentEncoding) {
       return this;
     }
 
-    Builder componentCount(int componentCount) {
+    Builder componentCount(Integer componentCount) {
       this.componentCount = componentCount;
       return this;
     }
@@ -155,7 +155,7 @@ public Builder owner(Acl.Entity owner) {
       return this;
     }
 
-    public Builder size(long size) {
+    public Builder size(Long size) {
       this.size = size;
       return this;
     }
@@ -190,34 +190,36 @@ public Builder metadata(Map metadata) {
       return this;
     }
 
-    public Builder generation(long generation) {
+    public Builder generation(Long generation) {
       this.generation = generation;
       return this;
     }
 
-    public Builder metageneration(long metageneration) {
+    public Builder metageneration(Long metageneration) {
       this.metageneration = metageneration;
       return this;
     }
 
-    public Builder deleteTime(long deleteTime) {
+    public Builder deleteTime(Long deleteTime) {
       this.deleteTime = deleteTime;
       return this;
     }
 
-    public Builder updateTime(long updateTime) {
+    public Builder updateTime(Long updateTime) {
       this.updateTime = updateTime;
       return this;
     }
 
     public Blob build() {
+      checkNotNull(bucket);
+      checkNotNull(name);
       return new Blob(this);
     }
   }
 
   private Blob(Builder builder) {
-    bucket = checkNotNull(builder.bucket);
-    name = checkNotNull(builder.name);
+    bucket = builder.bucket;
+    name = builder.name;
     id = builder.id;
     cacheControl = builder.cacheControl;
     contentEncoding = builder.contentEncoding;
@@ -264,7 +266,7 @@ public Acl.Entity owner() {
     return owner;
   }
 
-  public long size() {
+  public Long size() {
     return size;
   }
 
@@ -284,7 +286,7 @@ public String contentLanguage() {
     return contentEncoding;
   }
 
-  public int componentCount() {
+  public Integer componentCount() {
     return componentCount;
   }
 
@@ -312,19 +314,19 @@ public Map metadata() {
     return metadata;
   }
 
-  public long generation() {
+  public Long generation() {
     return generation;
   }
 
-  public long metageneration() {
+  public Long metageneration() {
     return metageneration;
   }
 
-  public long deleteTime() {
+  public Long deleteTime() {
     return deleteTime;
   }
 
-  public long updateTime() {
+  public Long updateTime() {
     return updateTime;
   }
 
@@ -364,27 +366,36 @@ public static Builder builder(String bucket, String name) {
 
   StorageObject toPb() {
     StorageObject storageObject = new StorageObject();
-    storageObject.setAcl(Lists.transform(acl,
-        new Function() {
-          @Override public ObjectAccessControl apply(Acl acl) {
-            return acl.toObjectPb();
-          }
-        }));
+    if (acl != null) {
+      storageObject.setAcl(Lists.transform(acl, new Function() {
+        @Override public ObjectAccessControl apply(Acl acl) {
+          return acl.toObjectPb();
+        }
+      }));
+    }
+    if (deleteTime != null) {
+      storageObject.setTimeDeleted(new DateTime(deleteTime));
+    }
+    if (updateTime != null) {
+      storageObject.setUpdated(new DateTime(updateTime));
+    }
+    if (size != null) {
+      storageObject.setSize(BigInteger.valueOf(size));
+    }
+    if (owner != null) {
+      storageObject.setOwner(new Owner().setEntity(owner.toPb()));
+    }
     storageObject.setBucket(bucket);
     storageObject.setCacheControl(cacheControl);
     storageObject.setContentEncoding(contentEncoding);
     storageObject.setCrc32c(crc32c);
     storageObject.setContentType(contentType);
-    storageObject.setTimeDeleted(new DateTime(deleteTime));
     storageObject.setGeneration(generation);
     storageObject.setMd5Hash(md5);
     storageObject.setMediaLink(mediaLink);
     storageObject.setMetadata(metadata);
     storageObject.setMetageneration(metageneration);
     storageObject.setName(name);
-    storageObject.setOwner(new Owner().setEntity(owner.toPb()));
-    storageObject.setUpdated(new DateTime(updateTime));
-    storageObject.setSize(BigInteger.valueOf(size));
     storageObject.setContentDisposition(contentDisposition);
     storageObject.setComponentCount(componentCount);
     storageObject.setContentLanguage(contentLanguage);
@@ -395,34 +406,44 @@ StorageObject toPb() {
   }
 
   static Blob fromPb(StorageObject storageObject) {
-    return new Builder()
-        .acl(Lists.transform(storageObject.getAcl(),
-            new Function() {
-              @Override public Acl apply(ObjectAccessControl objectAccessControl) {
-                return Acl.fromPb(objectAccessControl);
-              }
-            }))
+    Builder builder = new Builder()
         .bucket(storageObject.getBucket())
         .cacheControl(storageObject.getCacheControl())
         .contentEncoding(storageObject.getContentEncoding())
         .crc32c(storageObject.getCrc32c())
         .contentType(storageObject.getContentType())
-        .deleteTime(storageObject.getTimeDeleted().getValue())
         .generation(storageObject.getGeneration())
         .md5(storageObject.getMd5Hash())
         .mediaLink(storageObject.getMediaLink())
         .metadata(storageObject.getMetadata())
         .metageneration(storageObject.getMetageneration())
         .name(storageObject.getName())
-        .owner(Acl.Entity.fromPb(storageObject.getOwner().getEntity()))
-        .updateTime(storageObject.getUpdated().getValue())
-        .size(storageObject.getSize().longValue())
         .contentDisposition(storageObject.getContentDisposition())
         .componentCount(storageObject.getComponentCount())
         .contentLanguage(storageObject.getContentLanguage())
         .etag(storageObject.getEtag())
         .id(storageObject.getId())
-        .selfLink(storageObject.getSelfLink())
-        .build();
+        .selfLink(storageObject.getSelfLink());
+    if (storageObject.getTimeDeleted() != null) {
+      builder.deleteTime(storageObject.getTimeDeleted().getValue());
+    }
+    if (storageObject.getUpdated() != null) {
+      builder.updateTime(storageObject.getUpdated().getValue());
+    }
+    if (storageObject.getSize() != null) {
+      builder.size(storageObject.getSize().longValue());
+    }
+    if (storageObject.getOwner() != null) {
+      builder.owner(Acl.Entity.fromPb(storageObject.getOwner().getEntity()));
+    }
+    if (storageObject.getAcl() != null) {
+      builder.acl(Lists.transform(storageObject.getAcl(), new Function() {
+        @Override
+        public Acl apply(ObjectAccessControl objectAccessControl) {
+          return Acl.fromPb(objectAccessControl);
+        }
+      }));
+    }
+    return builder.build();
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 4771506a2a35..55586acc3cbb 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -28,7 +28,6 @@
 import com.google.api.services.storage.model.BucketAccessControl;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.common.base.Function;
-import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.storage.Acl.Entity;
@@ -44,16 +43,16 @@ public final class Bucket implements Serializable {
   private final String name;
   private final Acl.Entity owner;
   private final String selfLink;
-  private final boolean versioningEnabled;
+  private final Boolean versioningEnabled;
   private final String indexPage;
   private final String notFoundPage;
-  private final ImmutableList deleteRules;
+  private final List deleteRules;
   private final String etag;
-  private final long createTime;
-  private final long metageneration;
-  private final ImmutableList cors;
-  private final ImmutableList acl;
-  private final ImmutableList defaultAcl;
+  private final Long createTime;
+  private final Long metageneration;
+  private final List cors;
+  private final List acl;
+  private final List defaultAcl;
   private final Location location;
   private final StorageClass storageClass;
 
@@ -332,18 +331,18 @@ public final static class Builder {
     private String name;
     private Acl.Entity owner;
     private String selfLink;
-    private boolean versioningEnabled;
+    private Boolean versioningEnabled;
     private String indexPage;
     private String notFoundPage;
-    private ImmutableList deleteRules = ImmutableList.of();
+    private ImmutableList deleteRules;
     private StorageClass storageClass;
     private Location location;
     private String etag;
     private Long createTime;
     private Long metageneration;
-    private Iterable cors = ImmutableList.of();
-    private Iterable acl = ImmutableList.of();
-    private Iterable defaultAcl = ImmutableList.of();
+    private ImmutableList cors;
+    private ImmutableList acl;
+    private ImmutableList defaultAcl;
 
     private Builder() {
     }
@@ -368,7 +367,7 @@ Builder selfLink(String selfLink) {
       return this;
     }
 
-    public Builder versioningEnabled(boolean enable) {
+    public Builder versioningEnabled(Boolean enable) {
       this.versioningEnabled = enable;
       return this;
     }
@@ -414,7 +413,7 @@ Builder metageneration(Long metageneration) {
     }
 
     public Builder cors(Iterable cors) {
-      this.cors = cors;
+      this.cors = ImmutableList.copyOf(cors);
       return this;
     }
 
@@ -429,27 +428,28 @@ public Builder defaultAcl(Iterable acl) {
     }
 
     public Bucket build() {
+      checkNotNull(name);
       return new Bucket(this);
     }
   }
 
   private Bucket(Builder builder) {
     id = builder.id;
-    name = checkNotNull(builder.name);
+    name = builder.name;
     etag = builder.etag;
-    createTime = MoreObjects.firstNonNull(builder.createTime, 0L);
-    metageneration = MoreObjects.firstNonNull(builder.metageneration, 0L);
+    createTime = builder.createTime;
+    metageneration = builder.metageneration;
     location = builder.location;
     storageClass = builder.storageClass;
-    cors = ImmutableList.copyOf(builder.cors);
-    acl = ImmutableList.copyOf(builder.acl);
-    defaultAcl = ImmutableList.copyOf(builder.defaultAcl);
+    cors = builder.cors;
+    acl = builder.acl;
+    defaultAcl = builder.defaultAcl;
     owner = builder.owner;
     selfLink = builder.selfLink;
     versioningEnabled = builder.versioningEnabled;
     indexPage = builder.indexPage;
     notFoundPage = builder.notFoundPage;
-    deleteRules = ImmutableList.copyOf(builder.deleteRules);
+    deleteRules = builder.deleteRules;
   }
 
   public String id() {
@@ -468,7 +468,7 @@ public String selfLink() {
     return selfLink;
   }
 
-  public boolean versioningEnabled() {
+  public Boolean versioningEnabled() {
     return versioningEnabled;
   }
 
@@ -480,7 +480,7 @@ public String notFoundPage() {
     return notFoundPage;
   }
 
-  public ImmutableList deleteRules() {
+  public List deleteRules() {
     return deleteRules;
   }
 
@@ -488,11 +488,11 @@ public String etag() {
     return etag;
   }
 
-  public long createTime() {
+  public Long createTime() {
     return createTime;
   }
 
-  public long metageneration() {
+  public Long metageneration() {
     return metageneration;
   }
 
@@ -546,10 +546,10 @@ com.google.api.services.storage.model.Bucket toPb() {
     bucketPb.setId(id);
     bucketPb.setName(name);
     bucketPb.setEtag(etag);
-    if (createTime > 0) {
+    if (createTime != null) {
       bucketPb.setTimeCreated(new DateTime(createTime));
     }
-    if (metageneration > 0) {
+    if (metageneration != null) {
       bucketPb.setMetageneration(metageneration);
     }
     if (location != null) {
@@ -558,31 +558,44 @@ com.google.api.services.storage.model.Bucket toPb() {
     if (storageClass != null) {
       bucketPb.setStorageClass(storageClass.value());
     }
-    bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION));
-    bucketPb.setAcl(transform(acl, new Function() {
-      @Override public BucketAccessControl apply(Acl acl) {
-        return acl.toBucketPb();
-      }
-    }));
-    bucketPb.setDefaultObjectAcl(transform(defaultAcl,
-        new Function() {
-          @Override public ObjectAccessControl apply(Acl acl) {
-            return acl.toObjectPb();
-          }
-        }));
-    bucketPb.setOwner(new Owner().setEntity(owner.toPb()));
+    if (cors != null) {
+      bucketPb.setCors(transform(cors, Cors.TO_PB_FUNCTION));
+    }
+    if (acl != null) {
+      bucketPb.setAcl(transform(acl, new Function() {
+        @Override public BucketAccessControl apply(Acl acl) {
+          return acl.toBucketPb();
+        }
+      }));
+    }
+    if (defaultAcl != null) {
+      bucketPb.setDefaultObjectAcl(transform(defaultAcl, new Function() {
+        @Override public ObjectAccessControl apply(Acl acl) {
+          return acl.toObjectPb();
+        }
+      }));
+    }
+    if (owner != null) {
+      bucketPb.setOwner(new Owner().setEntity(owner.toPb()));
+    }
     bucketPb.setSelfLink(selfLink);
-    bucketPb.setVersioning(new Versioning().setEnabled(versioningEnabled));
-    Website website = new Website();
-    website.setMainPageSuffix(indexPage);
-    website.setNotFoundPage(notFoundPage);
-    bucketPb.setWebsite(website);
-    Lifecycle lifecycle = new Lifecycle();
-    lifecycle.setRule(transform(deleteRules, new Function() {
-      @Override public Rule apply(DeleteRule deleteRule) {
-        return deleteRule.toPb();
-      }
-    }));
+    if (versioningEnabled != null) {
+      bucketPb.setVersioning(new Versioning().setEnabled(versioningEnabled));
+    }
+    if (indexPage != null || notFoundPage != null) {
+      Website website = new Website();
+      website.setMainPageSuffix(indexPage);
+      website.setNotFoundPage(notFoundPage);
+      bucketPb.setWebsite(website);
+    }
+    if (deleteRules != null) {
+      Lifecycle lifecycle = new Lifecycle();
+      lifecycle.setRule(transform(deleteRules, new Function() {
+        @Override public Rule apply(DeleteRule deleteRule) {
+          return deleteRule.toPb();
+        }
+      }));
+    }
     return bucketPb;
   }
 
@@ -590,38 +603,46 @@ static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) {
     Builder builder = new Builder()
         .name(bucketPb.getName())
         .id(bucketPb.getId())
-        .createTime(bucketPb.getTimeCreated().getValue())
         .etag(bucketPb.getEtag())
         .metageneration(bucketPb.getMetageneration())
+        .createTime(bucketPb.getTimeCreated().getValue())
         .location(Location.of(bucketPb.getLocation()))
-        .storageClass(StorageClass.of(bucketPb.getStorageClass()))
-        .cors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION))
-        .acl(transform(bucketPb.getAcl(),
-            new Function() {
-              @Override public Acl apply(BucketAccessControl bucketAccessControl) {
-                return Acl.fromPb(bucketAccessControl);
-              }
-            }))
-        .defaultAcl(transform(bucketPb.getDefaultObjectAcl(),
-            new Function() {
-              @Override public Acl apply(ObjectAccessControl objectAccessControl) {
-                return Acl.fromPb(objectAccessControl);
-              }
-            }))
-        .owner(Entity.fromPb(bucketPb.getOwner().getEntity()))
         .selfLink(bucketPb.getSelfLink());
-    Versioning versioning = bucketPb.getVersioning();
-    if (versioning != null) {
-      builder.versioningEnabled(MoreObjects.firstNonNull(versioning.getEnabled(), Boolean.FALSE));
+    if (bucketPb.getStorageClass() != null) {
+      builder.storageClass(StorageClass.of(bucketPb.getStorageClass()));
+    }
+    if (bucketPb.getCors() != null) {
+      builder.cors(transform(bucketPb.getCors(), Cors.FROM_PB_FUNCTION));
+    }
+    if (bucketPb.getAcl() != null) {
+      builder.acl(transform(bucketPb.getAcl(), new Function() {
+        @Override public Acl apply(BucketAccessControl bucketAccessControl) {
+          return Acl.fromPb(bucketAccessControl);
+        }
+      }));
+    }
+    if (bucketPb.getDefaultObjectAcl() != null) {
+      builder.defaultAcl(transform(bucketPb.getDefaultObjectAcl(),
+          new Function() {
+            @Override
+            public Acl apply(ObjectAccessControl objectAccessControl) {
+              return Acl.fromPb(objectAccessControl);
+            }
+          }));
+    }
+    if (bucketPb.getOwner() != null) {
+      builder.owner(Entity.fromPb(bucketPb.getOwner().getEntity()));
+    }
+    if (bucketPb.getVersioning() != null) {
+      builder.versioningEnabled(bucketPb.getVersioning().getEnabled());
     }
     Website website = bucketPb.getWebsite();
     if (website != null) {
       builder.indexPage(website.getMainPageSuffix());
       builder.notFoundPage(website.getNotFoundPage());
     }
-    Lifecycle lifecycle = bucketPb.getLifecycle();
-    if (lifecycle != null) {
-      builder.deleteRules(transform(lifecycle.getRule(),
+    if (bucketPb.getLifecycle() != null && bucketPb.getLifecycle().getRule() != null) {
+      builder.deleteRules(transform(bucketPb.getLifecycle().getRule(),
           new Function() {
             @Override public DeleteRule apply(Rule rule) {
               return DeleteRule.fromPb(rule);
diff --git a/src/main/java/com/google/gcloud/storage/Option.java b/src/main/java/com/google/gcloud/storage/Option.java
new file mode 100644
index 000000000000..cdbf00015b72
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/Option.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Base class for Storage operation option
+ */
+public class Option implements Serializable {
+
+  private static final long serialVersionUID = -73199088766477208L;
+
+  private final String name;
+  private final Object value;
+
+  Option(String name, Object value) {
+    this.name = checkNotNull(name);
+    this.value = value;
+  }
+
+  public String name() {
+    return name;
+  }
+
+  public Object value() {
+    return value;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof Option)) {
+      return false;
+    }
+    Option other = (Option) obj;
+    return Objects.equals(name, other.name)
+        && Objects.equals(value, other.value);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(name, value);
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 5ce0c3d701bc..c78056c57168 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -17,6 +17,7 @@
 package com.google.gcloud.storage;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.gcloud.storage.Validator.checkBlobOptions;
 
 import com.google.common.collect.ImmutableList;
 import com.google.gcloud.Service;
@@ -29,7 +30,6 @@
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Objects;
 import java.util.Set;
 
 public interface StorageService extends Service {
@@ -57,42 +57,6 @@ String entry() {
     }
   }
 
-  class Option implements Serializable {
-
-    private static final long serialVersionUID = -73199088766477208L;
-
-    private final String name;
-    private final Object value;
-
-    Option(String name, Object value) {
-      this.name = checkNotNull(name);
-      this.value = value;
-    }
-
-    String name() {
-      return name;
-    }
-
-    Object value() {
-      return value;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-      if (!(obj instanceof Option)) {
-        return false;
-      }
-      Option other = (Option) obj;
-      return Objects.equals(name, other.name)
-          && Objects.equals(value, other.value);
-    }
-
-    @Override
-    public int hashCode() {
-      return Objects.hash(name, value);
-    }
-  }
-
   class BucketTargetOption extends Option {
 
     private static final long serialVersionUID = -5880204616982900975L;
@@ -140,7 +104,7 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
     }
 
     public static BlobTargetOption generationMath(boolean match) {
-      return new BlobTargetOption("ifMetagenerationMatch", match);
+      return new BlobTargetOption("ifGenerationMatch", match);
     }
 
     public static BlobTargetOption metagenerationMatch(boolean match) {
@@ -174,18 +138,18 @@ class ComposeRequest implements Serializable {
     private final Blob target;
     private final List targetOptions;
 
-    private static class SourceBlob implements Serializable {
+    static class SourceBlob implements Serializable {
 
       private static final long serialVersionUID = 4094962795951990439L;
 
       final String blob;
-      final String generation;
+      final Long generation;
 
       SourceBlob(String blob) {
         this(blob, null);
       }
 
-      SourceBlob(String blob, String generation) {
+      SourceBlob(String blob, Long generation) {
         this.blob = blob;
         this.generation = generation;
       }
@@ -214,8 +178,8 @@ public Builder addSource(String... blobs) {
         return addSource(Arrays.asList(blobs));
       }
 
-      public Builder addSource(String blob, String generation) {
-        sourceBlobs.add(new SourceBlob(blob, generation));
+      public Builder addSource(String blob, long matchGeneration) {
+        sourceBlobs.add(new SourceBlob(blob, matchGeneration));
         return this;
       }
 
@@ -230,14 +194,17 @@ public Builder targetOptions(BlobTargetOption... options) {
       }
 
       public ComposeRequest build() {
+        checkNotNull(bucket);
+        checkNotNull(target);
+        checkBlobOptions("Target", target, targetOptions);
         return new ComposeRequest(this);
       }
     }
 
     private ComposeRequest(Builder builder) {
-      sourceBucket = checkNotNull(builder.bucket);
+      sourceBucket = builder.bucket;
       sourceBlobs = ImmutableList.copyOf(builder.sourceBlobs);
-      target = checkNotNull(builder.target);
+      target = builder.target;
       targetOptions = ImmutableList.copyOf(builder.targetOptions);
     }
 
@@ -298,6 +265,10 @@ public Builder targetOptions(BlobTargetOption... options) {
       }
 
       public CopyRequest build() {
+        checkNotNull(source);
+        checkNotNull(target);
+        checkBlobOptions("Source", source, sourceOptions);
+        checkBlobOptions("Target", target, targetOptions);
         return new CopyRequest(this);
       }
     }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 78acb40d5b9a..f5c25443eeea 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,13 +16,16 @@
 
 package com.google.gcloud.storage;
 
+import com.google.api.services.storage.model.StorageObject;
 import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.spi.StorageRpc;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
+import java.util.List;
 
 final class StorageServiceImpl extends BaseService implements StorageService {
 
@@ -43,7 +46,7 @@ final class StorageServiceImpl extends BaseService implem
   @Override
   public Bucket create(Bucket bucket, BucketTargetOption... options) {
     try {
-      return Bucket.fromPb(storageRpc.create(bucket.toPb()));
+      return Bucket.fromPb(storageRpc.create(bucket.toPb(), options));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -52,7 +55,7 @@ public Bucket create(Bucket bucket, BucketTargetOption... options) {
   @Override
   public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) {
     try {
-      return Blob.fromPb(storageRpc.create(blob.toPb(), content));
+      return Blob.fromPb(storageRpc.create(blob.toPb(), content, options));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -61,7 +64,7 @@ public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) {
   @Override
   public Bucket get(String bucket, BucketSourceOption... options) {
     try {
-      return Bucket.fromPb(storageRpc.get(bucket));
+      return Bucket.fromPb(storageRpc.get(bucket, options));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -70,7 +73,7 @@ public Bucket get(String bucket, BucketSourceOption... options) {
   @Override
   public Blob get(String bucket, String blob, BlobSourceOption... options) {
     try {
-      return Blob.fromPb(storageRpc.get(bucket, blob));
+      return Blob.fromPb(storageRpc.get(bucket, blob, options));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -101,7 +104,7 @@ public Iterator list(String bucket, ListOptions settings) {
   @Override
   public Bucket update(Bucket bucket, BucketTargetOption... options) {
     try {
-      return Bucket.fromPb(storageRpc.patch(bucket.toPb()));
+      return Bucket.fromPb(storageRpc.patch(bucket.toPb(), options));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -110,7 +113,7 @@ public Bucket update(Bucket bucket, BucketTargetOption... options) {
   @Override
   public Blob update(Blob blob, BlobTargetOption... options) {
     try {
-      return Blob.fromPb(storageRpc.patch(blob.toPb()));
+      return Blob.fromPb(storageRpc.patch(blob.toPb(), options));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -119,7 +122,7 @@ public Blob update(Blob blob, BlobTargetOption... options) {
   @Override
   public void delete(Bucket bucket, BucketSourceOption... options) {
     try {
-      storageRpc.delete(bucket.toPb());
+      storageRpc.delete(bucket.toPb(), options);
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -128,7 +131,7 @@ public void delete(Bucket bucket, BucketSourceOption... options) {
   @Override
   public void delete(Blob blob, BlobSourceOption... options) {
     try {
-      storageRpc.delete(blob.toPb());
+      storageRpc.delete(blob.toPb(), options);
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -136,22 +139,26 @@ public void delete(Blob blob, BlobSourceOption... options) {
 
   @Override
   public Blob compose(ComposeRequest composeRequest) {
-    // todo: implement (consider having XXXRequest/XXXResponse for all SPI/Rpc requests
-// try {
-
-//
-//      return Blob.fromPb(storageRpc.compose(composeRequest.sourceBucket(),
-//          composeRequest.sourceBlobs(), composeRequest.target()));
-//    } catch (IOException ex) {
-//      throw new StorageServiceException(ex);
-//    }
-    throw new UnsupportedOperationException("bla");
+    List sources = Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size());
+    for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) {
+      sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob)
+          .generation(sourceBlob.generation)
+          .build()
+          .toPb());
+    }
+    try {
+      return Blob.fromPb(storageRpc.compose(sources, composeRequest.target().toPb(),
+          composeRequest.targetOptions()));
+    } catch (IOException ex) {
+      throw new StorageServiceException(ex);
+    }
   }
 
   @Override
   public Blob copy(CopyRequest copyRequest) {
     try {
-      return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.target().toPb()));
+      return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.sourceOptions(),
+          copyRequest.target().toPb(), copyRequest.targetOptions()));
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -160,7 +167,7 @@ public Blob copy(CopyRequest copyRequest) {
   @Override
   public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) {
     try {
-      return storageRpc.reader(blob.toPb());
+      return storageRpc.reader(blob.toPb(), options);
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
@@ -169,7 +176,7 @@ public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) {
   @Override
   public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) {
     try {
-      return storageRpc.writer(blob.toPb());
+      return storageRpc.writer(blob.toPb(), options);
     } catch (IOException ex) {
       throw new StorageServiceException(ex);
     }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index a1b61bcc00d4..10b54d79e413 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -30,6 +30,7 @@ public class StorageServiceOptions extends ServiceOptions {
   private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control";
   private static final Set SCOPES = ImmutableSet.of(GCS_SCOPE);
   private static final String DEFAULT_PATH_DELIMITER = "/";
+  private static final String PROJECT_ENV_NAME = "default_project_id";
 
   private final StorageRpc storageRpc;
   private final String project;
@@ -72,7 +73,7 @@ public StorageServiceOptions build() {
   private StorageServiceOptions(Builder builder) {
     super(builder);
     pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
-    project = builder.project != null ? builder.project :  getAppEngineProjectId();
+    project = builder.project != null ? builder.project : defaultProject();
     Preconditions.checkArgument(project != null, "Missing required project id");
     storageRpc = MoreObjects.firstNonNull(builder.storageRpc, ServiceRpcProvider.storage(this));
   }
@@ -99,6 +100,14 @@ public Builder toBuilder() {
     return new Builder(this);
   }
 
+  private static String defaultProject() {
+    String projectId = System.getProperty(PROJECT_ENV_NAME, System.getenv(PROJECT_ENV_NAME));
+    if (projectId == null) {
+      projectId = getAppEngineProjectId();
+    }
+    return projectId != null ? projectId : googleCloudProjectId();
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/src/main/java/com/google/gcloud/storage/Validator.java b/src/main/java/com/google/gcloud/storage/Validator.java
new file mode 100644
index 000000000000..0db8fd3be16e
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/Validator.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Utility to validate Storage type/values.
+ */
+public class Validator {
+
+  static void checkBlobOptions(String name, Blob blob, Iterable options) {
+    for (Option option : options) {
+      switch (option.name()) {
+        case "ifGenerationMatch":
+          checkArgument(blob.generation() != 0, "%s blob is missing generation", name);
+          break;
+        case "ifMetagenerationMatch":
+          checkArgument(blob.metageneration() != 0, "%s blob is missing metageneration", name);
+          break;
+      }
+    }
+  }
+}

From 577a3e10fdb831b3693490c3834242b6db4c4307 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 21 Apr 2015 08:55:40 -0700
Subject: [PATCH 170/732] fix retry helper for datastore and change the way we
 get a default instance of the service

---
 .../datastore/DatastoreServiceFactory.java    | 10 +++++--
 .../datastore/DatastoreServiceImpl.java       |  4 +--
 .../google/gcloud/datastore/package-info.java |  2 +-
 .../gcloud/examples/DatastoreExample.java     |  2 +-
 .../datastore/DatastoreServiceTest.java       | 29 ++++++++++++++++++-
 5 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
index ca8dfa7e2982..b2cd7c8eec11 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
@@ -27,9 +27,15 @@ public DatastoreService get(DatastoreServiceOptions options) {
       }
     };
 
-  public static DatastoreService getDefault(DatastoreServiceOptions options) {
-    return INSTANCE.get(options);
+  /**
+   * Returns the default factory instance.
+   */
+  public static DatastoreServiceFactory instance() {
+    return INSTANCE;
   }
 
+  /**
+   * Returns a {@code DatastoreService} for the given options.
+   */
   public abstract DatastoreService get(DatastoreServiceOptions options);
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 1b1bf81c3e2c..0958e920b7b6 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -60,8 +60,8 @@ public RetryResult afterEval(Exception exception, RetryResult retryResult) {
 
         @Override
         public RetryResult beforeEval(Exception exception) {
-          if (exception instanceof DatastoreServiceException) {
-            boolean retriable = ((DatastoreServiceException) exception).code().retriable();
+          if (exception instanceof DatastoreRpcException) {
+            boolean retriable = ((DatastoreRpcException) exception).retryable();
             return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
           }
           return null;
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index 06073f6f2329..5c4f6d945bcc 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -20,7 +20,7 @@
  * 
A simple usage example:
  * 
 {@code
  * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
- * DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
+ * DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
  * KeyFactory keyFactory = datastore.newKeyFactory().kind(kind);
  * Key key = keyFactory.newKey(keyName);
  * Entity entity = datastore.get(key);
diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index 7efd3af00b3b..d2e0aa06d9ae 100644
--- a/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -187,7 +187,7 @@ public static void main(String... args) {
           .namespace(NAMESPACE)
           .build();
       String name = args.length > 1 ? args[1] : System.getProperty("user.name");
-      datastore = DatastoreServiceFactory.getDefault(options);
+      datastore = DatastoreServiceFactory.instance().get(options);
       KeyFactory keyFactory = datastore.newKeyFactory().kind(USER_KIND);
       key = keyFactory.newKey(name);
       String actionName = args.length > 2 ? args[2].toLowerCase() : DEFAULT_ACTION;
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 943ac5705858..c9ff5d1632ab 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -25,12 +25,18 @@
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import com.google.api.services.datastore.DatastoreV1;
+import com.google.api.services.datastore.DatastoreV1.EntityResult;
 import com.google.common.collect.Iterators;
+import com.google.gcloud.RetryParams;
 import com.google.gcloud.datastore.Query.ResultType;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
+import com.google.gcloud.spi.DatastoreRpc;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
 
+import org.easymock.EasyMock;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -106,7 +112,7 @@ public void setUp() throws IOException, InterruptedException {
         .dataset(DATASET)
         .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
-    datastore = DatastoreServiceFactory.getDefault(options);
+    datastore = DatastoreServiceFactory.instance().get(options);
     StructuredQuery query = Query.keyQueryBuilder().build();
     QueryResults result = datastore.run(query);
     datastore.delete(Iterators.toArray(result, Key.class));
@@ -624,4 +630,25 @@ public void testKeyFactory() {
     assertEquals(KEY1, keyFactory.newKey("name"));
     assertEquals(Key.builder(KEY1).id(2).build(), keyFactory.newKey(2));
   }
+
+  @Test
+  public void testRetires() throws Exception {
+    DatastoreV1.LookupRequest requestPb =
+        DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build();
+    DatastoreV1.LookupResponse responsePb = DatastoreV1.LookupResponse.newBuilder()
+        .addFound(EntityResult.newBuilder().setEntity(ENTITY1.toPb())).build();
+    DatastoreRpc mock = EasyMock.createStrictMock(DatastoreRpc.class);
+    EasyMock.expect(mock.lookup(requestPb))
+        .andThrow(new DatastoreRpc.DatastoreRpcException(Reason.UNAVAILABLE))
+        .andReturn(responsePb);
+    EasyMock.replay(mock);
+    DatastoreServiceOptions options = this.options.toBuilder()
+        .retryParams(RetryParams.getDefaultInstance())
+        .datastoreRpc(mock)
+        .build();
+    DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
+    Entity entity = datastore.get(KEY1);
+    assertEquals(ENTITY1, entity);
+    EasyMock.verify(mock);
+  }
 }

From 894541860cdc6da9babd6ab1b3d66392f020d6bb Mon Sep 17 00:00:00 2001
From: Arie 
Date: Tue, 21 Apr 2015 08:56:34 -0700
Subject: [PATCH 171/732] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7bfa5acdf9a7..2da2f0f859fd 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ import com.google.gcloud.datastore.Key;
 import com.google.gcloud.datastore.KeyFactory;
 
 DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
-DatastoreService datastore = DatastoreServiceFactory.getDefault(options);
+DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
 KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
 Key key = keyFactory.newKey(keyName);
 Entity entity = datastore.get(key);

From 673fbda1f4ff607e90f7a165fc5c2467123a8929 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 21 Apr 2015 09:08:46 -0700
Subject: [PATCH 172/732] s/retriable/retryable/g

---
 src/main/java/com/google/gcloud/ExceptionHandler.java  |  2 +-
 .../gcloud/datastore/DatastoreServiceException.java    | 10 +++++-----
 .../google/gcloud/datastore/DatastoreServiceImpl.java  |  4 ++--
 .../datastore/DatastoreServiceExceptionTest.java       |  2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index 0e3f73d590fd..b1558f70bc18 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -187,7 +187,7 @@ private ExceptionHandler(Builder builder) {
     nonRetriableExceptions = builder.nonRetriableExceptions.build();
     Preconditions.checkArgument(
         Sets.intersection(retriableExceptions, nonRetriableExceptions).isEmpty(),
-        "Same exception was found in both retriable and non-retriable sets");
+        "Same exception was found in both retryable and non-retryable sets");
     for (Class exception : retriableExceptions) {
       addRetryInfo(new RetryInfo(exception, Interceptor.RetryResult.RETRY), retryInfo);
     }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
index 69094456a0c0..0d958005c22a 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
@@ -52,7 +52,7 @@ public enum Code {
     RESOURCE_EXHAUSTED(Reason.RESOURCE_EXHAUSTED),
     UNKNOWN(false, "Unknown failure", -1);
 
-    private final boolean retriable;
+    private final boolean retryable;
     private final String description;
     private final int httpStatus;
 
@@ -60,8 +60,8 @@ public enum Code {
       this(reason.retryable(), reason.description(), reason.httpStatus());
     }
 
-    Code(boolean retriable, String description, int httpStatus) {
-      this.retriable = retriable;
+    Code(boolean retryable, String description, int httpStatus) {
+      this.retryable = retryable;
       this.description = description;
       this.httpStatus = httpStatus;
     }
@@ -78,8 +78,8 @@ public int httpStatus() {
      * Returns {@code true} if this exception is transient and the same request could be retried.
      * For any retry it is highly recommended to apply an exponential backoff.
      */
-    public boolean retriable() {
-      return retriable;
+    public boolean retryable() {
+      return retryable;
     }
 
     DatastoreServiceException translate(DatastoreRpcException exception, String message) {
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
index 0958e920b7b6..d7712dd0bf15 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
@@ -61,8 +61,8 @@ public RetryResult afterEval(Exception exception, RetryResult retryResult) {
         @Override
         public RetryResult beforeEval(Exception exception) {
           if (exception instanceof DatastoreRpcException) {
-            boolean retriable = ((DatastoreRpcException) exception).retryable();
-            return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
+            boolean retryable = ((DatastoreRpcException) exception).retryable();
+            return retryable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
           }
           return null;
         }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
index d26cc246b3fb..2700d277b9a3 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retriable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retryable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file

From 8bcc7a2e9821125d4504c2dc495ccf3811086917 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 21 Apr 2015 11:35:32 -0700
Subject: [PATCH 173/732] work in progress

---
 .../google/gcloud/spi/DefaultStorageRpc.java  |   2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../google/gcloud/storage/StorageService.java |  45 +++-
 .../storage/StorageServiceException.java      |   2 +-
 .../gcloud/storage/StorageServiceFactory.java |   7 +-
 .../gcloud/storage/StorageServiceImpl.java    | 221 ++++++++++--------
 6 files changed, 175 insertions(+), 104 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index b61f96d23c06..41bff074c523 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.IOException;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory()
        .getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();

    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
  }

  @Override
  public List buckets() throws IOException {
    Buckets buckets = storage.buckets().list(options.project()).execute();
    return buckets.getItems();
  }

  @Override
  public List objects(String bucket, String prefix, String delimiter)
      throws IOException {
    return null;
  }

  @Override
  public Bucket get(String bucket) throws IOException {
    return storage.buckets().get(bucket).execute();
  }

  @Override
  public StorageObject get(String bucket, String object) throws IOException {
    return storage.objects().get(bucket, object).execute();
  }

  @Override
  public Bucket patch(Bucket bucket) throws IOException {
    return storage.buckets().patch(bucket.getName(), bucket).execute();
  }

  @Override
  public StorageObject patch(StorageObject storageObject) throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
  }

  @Override
  public void delete(String bucket, String object) throws IOException {
    storage.objects().delete(bucket, object).execute();
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false);
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(String bucket, Option... options) {
    try {
      return storage.buckets().get(bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(String bucket, String object, Option... options) {
    try {
      return storage.objects().get(bucket, object).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    try {
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 74b0d55aa02e..681f6206db96 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;

public interface StorageRpc {

  Bucket create(Bucket bucket, Option... options) throws IOException;
  StorageObject create(StorageObject object, ByteBuffer content, Option... options)
      throws IOException;
  Iterator list() throws IOException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws IOException;
  Bucket get(String bucket, Option... options) throws IOException;
  StorageObject get(String bucket, String object, Option... options) throws IOException;
  Bucket patch(Bucket bucket, Option... options) throws IOException;
  StorageObject patch(StorageObject storageObject, Option... options) throws IOException;
  void delete(Bucket bucket, Option... options) throws  IOException;
  void delete(StorageObject object, Option... options) throws IOException;
  StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws IOException;
  StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws IOException;
  BlobReadChannel reader(StorageObject from, Option... options) throws IOException;
  BlobWriteChannel writer(StorageObject to, Option... options) throws IOException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.List;

public interface StorageRpc {

  Bucket create(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject create(StorageObject object, byte[] content, Option... options)
      throws StorageServiceException;
  Iterator list() throws StorageServiceException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws StorageServiceException;
  Bucket get(String bucket, Option... options) throws StorageServiceException;
  StorageObject get(String bucket, String object, Option... options) throws StorageServiceException;
  Bucket patch(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject patch(StorageObject storageObject, Option... options)
      throws StorageServiceException;
  void delete(Bucket bucket, Option... options) throws  StorageServiceException;
  void delete(StorageObject object, Option... options) throws StorageServiceException;
  StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException;
  StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException;
  BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException;
  BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index c78056c57168..5271319b7246 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -23,7 +23,6 @@
 import com.google.gcloud.Service;
 
 import java.io.Serializable;
-import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
@@ -301,31 +300,73 @@ public static Builder builder() {
     }
   }
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Bucket create(Bucket bucket, BucketTargetOption... option);
 
-  Blob create(Blob blob, ByteBuffer content, BlobTargetOption... option);
+  /**
+   * @throws StorageServiceException upon failure
+   */
+  Blob create(Blob blob, byte[] content, BlobTargetOption... option);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Bucket get(String bucket, BucketSourceOption... options);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Blob get(String bucket, String blob, BlobSourceOption... options);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Iterator list();
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Iterator list(String bucket, ListOptions settings);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Bucket update(Bucket bucket, BucketTargetOption... option);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Blob update(Blob blob, BlobTargetOption... option);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   void delete(Bucket bucket, BucketSourceOption... option);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   void delete(Blob blob, BlobSourceOption... option);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Blob compose(ComposeRequest composeRequest);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   Blob copy(CopyRequest copyRequest);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   BlobReadChannel readFrom(Blob blob, BlobSourceOption... option);
 
+  /**
+   * @throws StorageServiceException upon failure
+   */
   BlobWriteChannel writeTo(Blob blob, BlobTargetOption... option);
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
index 8d181b182637..4a3438819250 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceException.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.IOException;

public class StorageServiceException extends RuntimeException {

  StorageServiceException(IOException ex) {
    super(ex);
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

/**
 * Storage service exception.
 *
 * @see Google Cloud Storage error codes
 */
public class StorageServiceException extends RuntimeException {

  private static final long serialVersionUID = -3748432005065428084L;

  private final int code;
  private final boolean retryable;

  public StorageServiceException(int code, String message, boolean retryable) {
    super(message);
    this.code = code;
    this.retryable = retryable;
  }

  /**
   * Returns the code associated with this exception.
   */
  public int code() {
    return code;
  }

  public boolean retryable() {
    return retryable;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
index 7f94334b3c08..2c2053b72d6b 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
@@ -21,14 +21,13 @@
 public abstract class StorageServiceFactory {
 
   private static final StorageServiceFactory INSTANCE = new StorageServiceFactory() {
-      @Override
-      public StorageService get(StorageServiceOptions options) {
+      @Override public StorageService get(StorageServiceOptions options) {
         return new StorageServiceImpl(options);
       }
     };
 
-  public static StorageService getDefault(StorageServiceOptions options) {
-    return INSTANCE.get(options);
+  public static StorageServiceFactory instance() {
+    return INSTANCE;
   }
 
   public abstract StorageService get(StorageServiceOptions options);
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index f5c25443eeea..fa0898bc5fc7 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,169 +16,200 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.gcloud.RetryHelper.runWithRetries;
+
 import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.MoreObjects;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.gcloud.BaseService;
+import com.google.gcloud.ExceptionHandler;
+import com.google.gcloud.ExceptionHandler.Interceptor;
+import com.google.gcloud.RetryParams;
 import com.google.gcloud.spi.StorageRpc;
 
-import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.Callable;
 
 final class StorageServiceImpl extends BaseService implements StorageService {
 
+  private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new Interceptor() {
+
+        private static final long serialVersionUID = -7758580330857881124L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return null;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof StorageServiceException) {
+            boolean retriable = ((StorageServiceException) exception).retryable();
+            return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
+          }
+          return null;
+        }
+      };
+  private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
+      .abortOn(RuntimeException.class)
+      .interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
+
   private final StorageRpc storageRpc;
+  private final RetryParams retryParams;
 
   StorageServiceImpl(StorageServiceOptions options) {
     super(options);
     storageRpc = options.storageRpc();
-
-    // todo: like Datastore distinct exception to retriable and non-retriable
-    //       https://cloud.google.com/storage/docs/json_api/v1/status-codes
-    // todo: Use retry helper on retriable failures
-
-    // todo: consider options
+    retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
     // todo: replace nulls with Value.asNull (per toPb)
   }
 
   @Override
-  public Bucket create(Bucket bucket, BucketTargetOption... options) {
-    try {
-      return Bucket.fromPb(storageRpc.create(bucket.toPb(), options));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Bucket create(Bucket bucket, final BucketTargetOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
+    return Bucket.fromPb(runWithRetries(
+        new Callable() {
+          @Override public com.google.api.services.storage.model.Bucket call() {
+            return storageRpc.create(bucketPb, options);
+          }
+        }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob create(Blob blob, ByteBuffer content, BlobTargetOption... options) {
-    try {
-      return Blob.fromPb(storageRpc.create(blob.toPb(), content, options));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Blob create(Blob blob, final byte[] content, final BlobTargetOption... options) {
+    final StorageObject blobPb = blob.toPb();
+    return Blob.fromPb(runWithRetries(new Callable() {
+          @Override public StorageObject call() {
+            return storageRpc.create(blobPb, content, options);
+          }
+        }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Bucket get(String bucket, BucketSourceOption... options) {
-    try {
-      return Bucket.fromPb(storageRpc.get(bucket, options));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Bucket get(final String bucket, final BucketSourceOption... options) {
+    return Bucket.fromPb(runWithRetries(
+        new Callable() {
+          @Override public com.google.api.services.storage.model.Bucket call() {
+            return storageRpc.get(bucket, options);
+          }
+        }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob get(String bucket, String blob, BlobSourceOption... options) {
-    try {
-      return Blob.fromPb(storageRpc.get(bucket, blob, options));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Blob get(final String bucket, final String blob, final BlobSourceOption... options) {
+    return Blob.fromPb(runWithRetries(new Callable() {
+      @Override public StorageObject call() {
+        return storageRpc.get(bucket, blob, options);
+      }
+    }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
   public Iterator list() {
-    try {
-      return Iterators.transform(storageRpc.list(), Bucket.FROM_PB_FUNCTION);
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+    return Iterators.transform(runWithRetries(
+        new Callable>() {
+          @Override public Iterator call() {
+            return storageRpc.list();
+          }
+        }, retryParams, EXCEPTION_HANDLER),
+        Bucket.FROM_PB_FUNCTION);
   }
 
   @Override
   public Iterator list(String bucket, ListOptions settings) {
-    try {
-      String delimiter = settings.recursive() ? options().pathDelimiter() : null;
-      return Iterators.transform(
-          storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(),
-              settings.includeOlderVersions(), settings.maxResults()),
-          Blob.FROM_PB_FUNCTION);
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+    // todo implement paging (with retries) with if limit is not given or > X
+    String delimiter = settings.recursive() ? options().pathDelimiter() : null;
+    return Iterators.transform(
+        storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(),
+            settings.includeOlderVersions(), settings.maxResults()),
+        Blob.FROM_PB_FUNCTION);
   }
 
   @Override
-  public Bucket update(Bucket bucket, BucketTargetOption... options) {
-    try {
-      return Bucket.fromPb(storageRpc.patch(bucket.toPb(), options));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Bucket update(Bucket bucket, final BucketTargetOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
+    return Bucket.fromPb(runWithRetries(
+        new Callable() {
+          @Override public com.google.api.services.storage.model.Bucket call() {
+            return storageRpc.patch(bucketPb, options);
+          }
+        }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob update(Blob blob, BlobTargetOption... options) {
-    try {
-      return Blob.fromPb(storageRpc.patch(blob.toPb(), options));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Blob update(Blob blob, final BlobTargetOption... options) {
+    final StorageObject storageObject = blob.toPb();
+    return Blob.fromPb(runWithRetries(new Callable() {
+      @Override public StorageObject call() {
+        return storageRpc.patch(storageObject, options);
+      }
+    }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public void delete(Bucket bucket, BucketSourceOption... options) {
-    try {
-      storageRpc.delete(bucket.toPb(), options);
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public void delete(Bucket bucket, final BucketSourceOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
+    runWithRetries(new Callable() {
+      @Override public Void call() {
+        storageRpc.delete(bucketPb, options);
+        return null;
+      }
+    }, retryParams, EXCEPTION_HANDLER);
   }
 
   @Override
-  public void delete(Blob blob, BlobSourceOption... options) {
-    try {
-      storageRpc.delete(blob.toPb(), options);
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public void delete(Blob blob, final BlobSourceOption... options) {
+    final StorageObject storageObject = blob.toPb();
+    runWithRetries(new Callable() {
+      @Override public Void call() {
+        storageRpc.delete(storageObject, options);
+        return null;
+      }
+    }, retryParams, EXCEPTION_HANDLER);
   }
 
   @Override
-  public Blob compose(ComposeRequest composeRequest) {
-    List sources = Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size());
+  public Blob compose(final ComposeRequest composeRequest) {
+    final List sources =
+        Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size());
     for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) {
       sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob)
           .generation(sourceBlob.generation)
           .build()
           .toPb());
     }
-    try {
-      return Blob.fromPb(storageRpc.compose(sources, composeRequest.target().toPb(),
-          composeRequest.targetOptions()));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+    final StorageObject target = composeRequest.target().toPb();
+    return Blob.fromPb(runWithRetries(new Callable() {
+      @Override public StorageObject call() {
+        return storageRpc.compose(sources, target, composeRequest.targetOptions());
+      }
+    }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob copy(CopyRequest copyRequest) {
-    try {
-      return Blob.fromPb(storageRpc.copy(copyRequest.source().toPb(), copyRequest.sourceOptions(),
-          copyRequest.target().toPb(), copyRequest.targetOptions()));
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+  public Blob copy(final CopyRequest copyRequest) {
+    final StorageObject source = copyRequest.source().toPb();
+    final StorageObject target = copyRequest.target().toPb();
+    return Blob.fromPb(runWithRetries(new Callable() {
+      @Override public StorageObject call() {
+        return storageRpc.copy(source, copyRequest.sourceOptions(), target,
+            copyRequest.targetOptions());
+      }
+    }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
   public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) {
-    try {
-      return storageRpc.reader(blob.toPb(), options);
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+    // todo: Use retry helper on retriable failures
+    return storageRpc.reader(blob.toPb(), options);
   }
 
   @Override
   public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) {
-    try {
-      return storageRpc.writer(blob.toPb(), options);
-    } catch (IOException ex) {
-      throw new StorageServiceException(ex);
-    }
+    // todo: Use retry helper on retriable failures
+    return storageRpc.writer(blob.toPb(), options);
   }
 }

From 93e265de9eedc6341ae0983634347c96db3cd6ec Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 21 Apr 2015 11:42:59 -0700
Subject: [PATCH 174/732] work in progress

---
 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 41bff074c523..7ae46db6cc99 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false);
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(String bucket, Option... options) {
    try {
      return storage.buckets().get(bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(String bucket, String object, Option... options) {
    try {
      return storage.objects().get(bucket, object).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    try {
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false);
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(String bucket, Option... options) {
    try {
      return storage.buckets().get(bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(String bucket, String object, Option... options) {
    try {
      return storage.objects().get(bucket, object).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return storage.objects()
          .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file

From 9edcd316d3fc0dac4cc2e10501eeb558d726e767 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 21 Apr 2015 17:35:00 -0700
Subject: [PATCH 175/732] work in progress

---
 .../gcloud/examples/StorageExample.java       | 171 +++++++++++++-----
 .../google/gcloud/spi/DefaultStorageRpc.java  |   2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../java/com/google/gcloud/storage/Blob.java  |   6 +-
 .../com/google/gcloud/storage/Bucket.java     |   4 +
 .../google/gcloud/storage/ListOptions.java    |   2 +-
 .../google/gcloud/storage/StorageService.java |   4 +-
 .../gcloud/storage/StorageServiceImpl.java    |  10 +-
 8 files changed, 141 insertions(+), 60 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index f670cc7ac1db..3c018f40b7d2 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -18,12 +18,14 @@
 
 import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.Bucket;
+import com.google.gcloud.storage.ListOptions;
 import com.google.gcloud.storage.StorageService;
 import com.google.gcloud.storage.StorageServiceFactory;
 import com.google.gcloud.storage.StorageServiceOptions;
 
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 
 /**
@@ -36,87 +38,156 @@
  * 
  • compile using maven - {@code mvn compile}
    • 
  * 
  • run using maven - {@code mvn exec:java
  * -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
- * -Dexec.args=" [] [list|get |put  []|delete ...]"}
    • 
+ * -Dexec.args="list []|info [ []]|get  |upload   []|delete  "}
  * 
  */
 public class StorageExample {
 
-  private static final String DEFAULT_FOLDER = "_STORAGE_EXAMPLE";
-  private static final String DEFAULT_ACTION = "list";
   private static final Map ACTIONS = new HashMap<>();
 
-  private static abstract class StorageAction {
+  private static abstract class StorageAction {
 
-    abstract void run(StorageService storage, Bucket bucket, String folder, String... args);
+    abstract void run(StorageService storage, T request);
 
-    protected String getRequiredParams() {
+    abstract T parse(String... args);
+
+    protected String params() {
       return "";
     }
+  }
+
+  private static abstract class BlobAction extends StorageAction {
 
-    protected String fullPath(String folder, String file) {
-      StringBuilder stringBuilder = new StringBuilder(folder);
-      if (!folder.endsWith("/")) {
-        stringBuilder.append('/');
+    @Override
+    Blob parse(String... args) {
+      if (args.length != 2) {
+        throw new IllegalArgumentException();
       }
-      return stringBuilder.append(file).toString();
+      return Blob.of(args[0], args[1]);
+    }
+
+    @Override
+    public String params() {
+      return " ";
     }
   }
 
-  private static class DeleteAction extends StorageAction {
+  private static class InfoAction extends BlobAction {
+    @Override
+    public void run(StorageService storage, Blob blob) {
+      if (blob.name().isEmpty()) {
+        System.out.println(storage.get(Bucket.of(blob.bucket())));
+      } else {
+        System.out.println(storage.get(blob));
+      }
+    }
+
     @Override
-    public void run(StorageService storage, Bucket bucket, String folder, String... args) {
-      for (String file : args) {
-        storage.delete(Blob.builder(bucket, fullPath(folder, file)).build());
+    Blob parse(String... args) {
+      if (args.length < 2) {
+        return Blob.of(args[0], "");
       }
+      return super.parse(args);
     }
 
     @Override
-    public String getRequiredParams() {
-      return "...";
+    public String params() {
+      return " []";
     }
   }
 
-  static {
-    ACTIONS.put("delete", new DeleteAction());
-    // todo: implement list, get and put
+  private static class DeleteAction extends BlobAction {
+    @Override
+    public void run(StorageService storage, Blob blob) {
+      storage.delete(blob);
+    }
+
+    @Override
+    public String params() {
+      return " ";
+    }
   }
 
-  public static void main(String... args) {
-    StorageAction action = null;
-    StorageService storage = null;
-    Bucket bucket = null;
-    String folder = DEFAULT_FOLDER;
-    if (args.length > 0) {
-      String bucketName = args[0];
-      String actionName = DEFAULT_ACTION;
-      if (args.length > 1) {
-        folder = args[1];
-        if (args.length > 2) {
-          actionName = args[2].toLowerCase();
-        }
-        storage = StorageServiceFactory.getDefault(StorageServiceOptions.builder().build());
-        bucket = storage.get(bucketName);
+  private static class ListAction extends StorageAction {
+
+    @Override
+    String parse(String... args) {
+      if (args.length == 0) {
+        return null;
+      }
+      if (args.length == 1) {
+        return args[0];
       }
-      action = ACTIONS.get(actionName);
+      throw new IllegalArgumentException();
     }
 
-    if (action == null) {
-      StringBuilder actionAndParams = new StringBuilder();
-      for (Map.Entry entry : ACTIONS.entrySet()) {
-        actionAndParams.append(entry.getKey());
-        String param = entry.getValue().getRequiredParams();
-        if (param != null && !param.isEmpty()) {
-          actionAndParams.append(' ').append(param);
+    @Override
+    public void run(StorageService storage, String bucket) {
+      if (bucket == null) {
+        Iterator buckets = storage.list();
+        while (buckets.hasNext()) {
+          System.out.println(buckets.next());
+        }
+      } else {
+        Iterator blobs = storage.list(bucket, ListOptions.of());
+        while (blobs.hasNext()) {
+          System.out.println(blobs.next());
         }
-        actionAndParams.append('|');
       }
-      actionAndParams.setLength(actionAndParams.length() - 1);
-      System.out.printf("Usage: %s bucket [] [%s]%n",
-          StorageExample.class.getSimpleName(), actionAndParams);
-      return;
     }
 
-    args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{};
-    action.run(storage, bucket, folder, args);
+    @Override
+    public String params() {
+      return "[]";
+    }
+  }
+
+  static {
+    ACTIONS.put("info", new InfoAction());
+    ACTIONS.put("delete", new DeleteAction());
+    ACTIONS.put("list", new ListAction());
+    //ACTIONS.put("upload", new UploadAction());
+    // todo: implement get
+  }
+
+  public static void printUsage() {
+    StringBuilder actionAndParams = new StringBuilder();
+    for (Map.Entry entry : ACTIONS.entrySet()) {
+      actionAndParams.append(entry.getKey());
+
+      String param = entry.getValue().params();
+      if (param != null && !param.isEmpty()) {
+        actionAndParams.append(' ').append(param);
+      }
+      actionAndParams.append('|');
+    }
+    actionAndParams.setLength(actionAndParams.length() - 1);
+    System.out.printf("Usage: %s [%s]%n",
+        StorageExample.class.getSimpleName(), actionAndParams);
+  }
+  public static void main(String... args) {
+    if (args.length == 0) {
+      System.out.println("Missing required action");
+      printUsage();
+      return;
+    }
+    StorageAction action = ACTIONS.get(args[0]);
+    if (action == null) {
+      System.out.println("Unrecognized action '" + args[0] + "'");
+      printUsage();
+      return;
+    }
+    StorageService storage =
+        StorageServiceFactory.instance().get(StorageServiceOptions.builder().build());
+    args = args.length > 1 ? Arrays.copyOfRange(args, 1, args.length): new String []{};
+    Object request;
+    try {
+      request = action.parse(args);
+    } catch (Exception ex) {
+      System.out.println("Invalid input for action '" + args[0] + "'");
+      System.out.println("Expected: " + action.params());
+      return;
+    }
+    action.run(storage, request);
   }
 }
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 7ae46db6cc99..e2ad7e19b431 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false);
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(String bucket, Option... options) {
    try {
      return storage.buckets().get(bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(String bucket, String object, Option... options) {
    try {
      return storage.objects().get(bucket, object).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return storage.objects()
          .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false);
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Option... options) {
    try {
      return storage.buckets().get(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Option... options) {
    try {
      return storage.objects().get(object.getBucket(), object.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return storage.objects()
          .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 681f6206db96..d051438244d2 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.List;

public interface StorageRpc {

  Bucket create(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject create(StorageObject object, byte[] content, Option... options)
      throws StorageServiceException;
  Iterator list() throws StorageServiceException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws StorageServiceException;
  Bucket get(String bucket, Option... options) throws StorageServiceException;
  StorageObject get(String bucket, String object, Option... options) throws StorageServiceException;
  Bucket patch(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject patch(StorageObject storageObject, Option... options)
      throws StorageServiceException;
  void delete(Bucket bucket, Option... options) throws  StorageServiceException;
  void delete(StorageObject object, Option... options) throws StorageServiceException;
  StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException;
  StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException;
  BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException;
  BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.List;

public interface StorageRpc {

  Bucket create(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject create(StorageObject object, byte[] content, Option... options)
      throws StorageServiceException;
  Iterator list() throws StorageServiceException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws StorageServiceException;
  Bucket get(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject get(StorageObject object, Option... options) throws StorageServiceException;
  Bucket patch(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject patch(StorageObject storageObject, Option... options)
      throws StorageServiceException;
  void delete(Bucket bucket, Option... options) throws  StorageServiceException;
  void delete(StorageObject object, Option... options) throws StorageServiceException;
  StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException;
  StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException;
  BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException;
  BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index 04553a8c744e..0e7496a72a06 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -356,7 +356,11 @@ public Builder toBuilder() {
         .selfLink(selfLink);
   }
 
-  public static Builder builder(Bucket bucket, String name) {
+  public static Blob of(String bucket, String name) {
+    return builder(bucket, name).build();
+  }
+
+    public static Builder builder(Bucket bucket, String name) {
     return builder(bucket.name(), name);
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 55586acc3cbb..6bfe63e6bd9c 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -536,6 +536,10 @@ public Builder toBuilder() {
         .deleteRules(deleteRules);
   }
 
+  public static Bucket of(String name) {
+    return builder(name).build();
+  }
+
   public static Builder builder(String name) {
     return new Builder().name(name);
   }
diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java
index 06637ed6aa3b..05257babca00 100644
--- a/src/main/java/com/google/gcloud/storage/ListOptions.java
+++ b/src/main/java/com/google/gcloud/storage/ListOptions.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    public Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    private Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }

    public ListOptions build() {
      return new ListOptions(this);
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }

  public static ListOptions of() {
    return builder().build();
  }

  public static Builder builder() {
    return new Builder();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 5271319b7246..ca3752e08b28 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -313,12 +313,12 @@ public static Builder builder() {
   /**
    * @throws StorageServiceException upon failure
    */
-  Bucket get(String bucket, BucketSourceOption... options);
+  Bucket get(Bucket bucket, BucketSourceOption... options);
 
   /**
    * @throws StorageServiceException upon failure
    */
-  Blob get(String bucket, String blob, BlobSourceOption... options);
+  Blob get(Blob blob, BlobSourceOption... options);
 
   /**
    * @throws StorageServiceException upon failure
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index fa0898bc5fc7..7d653a7fb25a 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -89,20 +89,22 @@ public Blob create(Blob blob, final byte[] content, final BlobTargetOption... op
   }
 
   @Override
-  public Bucket get(final String bucket, final BucketSourceOption... options) {
+  public Bucket get(Bucket bucket, final BucketSourceOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
     return Bucket.fromPb(runWithRetries(
         new Callable() {
           @Override public com.google.api.services.storage.model.Bucket call() {
-            return storageRpc.get(bucket, options);
+            return storageRpc.get(bucketPb, options);
           }
         }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob get(final String bucket, final String blob, final BlobSourceOption... options) {
+  public Blob get(Blob blob, final BlobSourceOption... options) {
+    final StorageObject storedObject = blob.toPb();
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override public StorageObject call() {
-        return storageRpc.get(bucket, blob, options);
+        return storageRpc.get(storedObject, options);
       }
     }, retryParams, EXCEPTION_HANDLER));
   }

From fcf4cff736d3bb81096e4fe92830e7ccdf647775 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 21 Apr 2015 20:18:37 -0700
Subject: [PATCH 176/732] work in progress

---
 .../gcloud/examples/StorageExample.java       | 91 +++++++++++++++----
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../java/com/google/gcloud/storage/Acl.java   |  4 +-
 .../java/com/google/gcloud/storage/Blob.java  |  6 ++
 .../com/google/gcloud/storage/Bucket.java     |  6 ++
 5 files changed, 87 insertions(+), 22 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 3c018f40b7d2..2fbaee752373 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,13 +16,11 @@
 
 package com.google.gcloud.examples;
 
-import com.google.gcloud.storage.Blob;
-import com.google.gcloud.storage.Bucket;
-import com.google.gcloud.storage.ListOptions;
-import com.google.gcloud.storage.StorageService;
-import com.google.gcloud.storage.StorageServiceFactory;
-import com.google.gcloud.storage.StorageServiceOptions;
+import com.google.gcloud.storage.*;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -38,7 +36,7 @@
  * 
  • compile using maven - {@code mvn compile}
    • 
  * 
  • run using maven - {@code mvn exec:java
  * -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
- * -Dexec.args="list []|info [ []]|get  |upload   []|delete  "}
    • 
+ * -Dexec.args="project_id list []|info [ []]|get  |upload   []|delete  "}
  * 
  */
 public class StorageExample {
@@ -47,15 +45,39 @@ public class StorageExample {
 
   private static abstract class StorageAction {
 
-    abstract void run(StorageService storage, T request);
+    abstract void run(StorageService storage, T request) throws Exception;
 
-    abstract T parse(String... args);
+    abstract T parse(String... args) throws IllegalArgumentException;
 
     protected String params() {
       return "";
     }
   }
 
+  private static class Tuple {
+
+    private final X x;
+    private final Y y;
+
+    private Tuple(X x, Y y) {
+      this.x = x;
+      this.y = y;
+    }
+
+    static  Tuple of(X x, Y y) {
+      return new Tuple<>(x, y);
+    }
+
+    X first() {
+      return x;
+    }
+
+    Y second() {
+      return y;
+    }
+  }
+
+
   private static abstract class BlobAction extends StorageAction {
 
     @Override
@@ -142,11 +164,39 @@ public String params() {
     }
   }
 
+  private static class UploadAction extends StorageAction> {
+    @Override
+    public void run(StorageService storage, Tuple tuple) throws Exception {
+      if (Files.size(tuple.first()) > 1_000_000) {
+        // todo upload via streaming API
+        throw new IllegalArgumentException("file is too big");
+      } else {
+        byte[] bytes = Files.readAllBytes(tuple.first());
+        System.out.println(storage.create(tuple.second(), bytes));
+      }
+    }
+
+    @Override
+    Tuple parse(String... args) {
+      if (args.length < 2 || args.length > 3) {
+        throw new IllegalArgumentException();
+      }
+      Path path = Paths.get(args[0]);
+      String blob = args.length < 3 ? path.getFileName().toString() : args[2];
+      return Tuple.of(path, Blob.of(args[1], blob));
+    }
+
+    @Override
+    public String params() {
+      return "  []";
+    }
+  }
+
   static {
     ACTIONS.put("info", new InfoAction());
     ACTIONS.put("delete", new DeleteAction());
     ACTIONS.put("list", new ListAction());
-    //ACTIONS.put("upload", new UploadAction());
+    ACTIONS.put("upload", new UploadAction());
     // todo: implement get
   }
 
@@ -165,26 +215,29 @@ public static void printUsage() {
     System.out.printf("Usage: %s [%s]%n",
         StorageExample.class.getSimpleName(), actionAndParams);
   }
-  public static void main(String... args) {
-    if (args.length == 0) {
-      System.out.println("Missing required action");
+
+  @SuppressWarnings("unchecked")
+  public static void main(String... args) throws Exception {
+    if (args.length < 2) {
+      System.out.println("Missing required project id and action");
       printUsage();
       return;
     }
-    StorageAction action = ACTIONS.get(args[0]);
+    String projectId = args[0];
+    StorageAction action = ACTIONS.get(args[1]);
     if (action == null) {
-      System.out.println("Unrecognized action '" + args[0] + "'");
+      System.out.println("Unrecognized action '" + args[1] + "'");
       printUsage();
       return;
     }
-    StorageService storage =
-        StorageServiceFactory.instance().get(StorageServiceOptions.builder().build());
-    args = args.length > 1 ? Arrays.copyOfRange(args, 1, args.length): new String []{};
+    StorageServiceOptions options = StorageServiceOptions.builder().project(args[0]).build();
+    StorageService storage = StorageServiceFactory.instance().get(options);
+    args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{};
     Object request;
     try {
       request = action.parse(args);
     } catch (Exception ex) {
-      System.out.println("Invalid input for action '" + args[0] + "'");
+      System.out.println("Invalid input for action '" + args[1] + "'");
       System.out.println("Expected: " + action.params());
       return;
     }
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index e2ad7e19b431..33f3d33924ab 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = transport.createRequestFactory().getInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated = new StorageServiceException(0, exception.getMessage(), false);
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Option... options) {
    try {
      return storage.buckets().get(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Option... options) {
    try {
      return storage.objects().get(object.getBucket(), object.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return storage.objects()
          .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.*;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      // todo: based on code consider if error is retryable
      translated = new StorageServiceException(details.getCode(), details.getMessage(), false);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Option... options) {
    try {
      return storage.buckets().get(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Option... options) {
    try {
      return storage.objects().get(object.getBucket(), object.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return storage.objects()
          .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 48bd6c787718..cb79e158d5f0 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -84,8 +84,8 @@ static Entity fromPb(String entity) {
         return new Group(entity.substring(6));
       }
       if (entity.startsWith("project-")) {
-        int idx = entity.indexOf('-', 9);
-        String team = entity.substring(9, idx);
+        int idx = entity.indexOf('-', 8);
+        String team = entity.substring(8, idx);
         String projectId = entity.substring(idx + 1);
         return new Project(Project.ProjectRole.valueOf(team.toUpperCase()), projectId);
       }
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index 0e7496a72a06..e6b6b150ced4 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -23,6 +23,7 @@
 import com.google.api.services.storage.model.StorageObject;
 import com.google.api.services.storage.model.StorageObject.Owner;
 import com.google.common.base.Function;
+import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -356,6 +357,11 @@ public Builder toBuilder() {
         .selfLink(selfLink);
   }
 
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(this).add("bucket", bucket).add("name", name).toString();
+  }
+
   public static Blob of(String bucket, String name) {
     return builder(bucket, name).build();
   }
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 6bfe63e6bd9c..1c3077e41bb6 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -28,6 +28,7 @@
 import com.google.api.services.storage.model.BucketAccessControl;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.common.base.Function;
+import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.storage.Acl.Entity;
@@ -536,6 +537,11 @@ public Builder toBuilder() {
         .deleteRules(deleteRules);
   }
 
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(this).add("name", name).toString();
+  }
+
   public static Bucket of(String name) {
     return builder(name).build();
   }

From 8bd611cc80dcbc9b9a4224709146c90ae347647c Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 22 Apr 2015 17:37:14 -0700
Subject: [PATCH 177/732] work in progress

---
 .../com/google/gcloud/AuthCredentials.java    |  18 +-
 .../com/google/gcloud/ExceptionHandler.java   |  29 ++-
 .../java/com/google/gcloud/RetryHelper.java   |  10 +-
 .../java/com/google/gcloud/RetryParams.java   |   8 +-
 .../com/google/gcloud/ServiceOptions.java     |   2 +-
 .../gcloud/examples/StorageExample.java       |  41 ++--
 .../google/gcloud/spi/DefaultStorageRpc.java  |   2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../google/gcloud/spi/StorageRpcFactory.java  |   2 +-
 .../java/com/google/gcloud/storage/Acl.java   |  14 +-
 .../java/com/google/gcloud/storage/Blob.java  |  24 ++-
 .../gcloud/storage/BlobReadChannel.java       |   4 +-
 .../gcloud/storage/BlobWriteChannel.java      |   6 +-
 .../com/google/gcloud/storage/Bucket.java     |  27 +--
 .../java/com/google/gcloud/storage/Cors.java  |  15 +-
 .../google/gcloud/storage/ListOptions.java    |   2 +-
 .../com/google/gcloud/storage/Option.java     |  31 ++-
 .../google/gcloud/storage/StorageService.java |  60 +++---
 .../storage/StorageServiceException.java      |   2 +-
 .../gcloud/storage/StorageServiceFactory.java |   9 +-
 .../gcloud/storage/StorageServiceImpl.java    | 183 ++++++++++++------
 .../gcloud/storage/StorageServiceOptions.java |   3 +-
 .../com/google/gcloud/storage/Validator.java  |  22 ++-
 .../google/gcloud/storage/package-info.java   |   1 +
 .../google/gcloud/ExceptionHandlerTest.java   |   3 +-
 25 files changed, 314 insertions(+), 206 deletions(-)

diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java
index 0514c26a4e7e..3e6c3651c576 100644
--- a/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -42,8 +42,8 @@ public abstract class AuthCredentials {
   private static class AppEngineAuthCredentials extends AuthCredentials {
 
     @Override
-    protected HttpRequestInitializer httpRequestInitializer(
-        HttpTransport transport, Set scopes) {
+    protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
+        Set scopes) {
       return new AppIdentityCredential(scopes);
     }
   }
@@ -85,8 +85,8 @@ public static AuthCredentials createForAppEngine() {
     return new AppEngineAuthCredentials();
   }
 
-  public static AuthCredentials createForComputeEngine()
-      throws IOException, GeneralSecurityException {
+  public static AuthCredentials createForComputeEngine() throws IOException,
+      GeneralSecurityException {
     final ComputeCredential cred = getComputeCredential();
     return new AuthCredentials() {
       @Override
@@ -100,10 +100,12 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
   /**
    * Returns the Application Default Credentials.
    *
-   * 
    Returns the Application Default Credentials which are credentials that identify and
-   * authorize the whole application. This is the built-in service account if running on Google
-   * Compute Engine or the credentials file from the path in the environment variable
-   * GOOGLE_APPLICATION_CREDENTIALS.
    
+   * 
    
+   * Returns the Application Default Credentials which are credentials that identify and authorize
+   * the whole application. This is the built-in service account if running on Google Compute Engine
+   * or the credentials file from the path in the environment variable
+   * GOOGLE_APPLICATION_CREDENTIALS.
+   * 
    
    *
    * @return the credentials instance.
    * @throws IOException if the credentials cannot be created in the current environment.
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/src/main/java/com/google/gcloud/ExceptionHandler.java
index 0e3f73d590fd..e4799934fa9c 100644
--- a/src/main/java/com/google/gcloud/ExceptionHandler.java
+++ b/src/main/java/com/google/gcloud/ExceptionHandler.java
@@ -49,8 +49,7 @@ public interface Interceptor extends Serializable {
 
     enum RetryResult {
 
-      RETRY(true),
-      ABORT(false);
+      RETRY(true), ABORT(false);
 
       private final boolean booleanValue;
 
@@ -67,9 +66,9 @@ boolean booleanValue() {
      * This method is called before exception evaluation and could short-circuit the process.
      *
      * @param exception the exception that is being evaluated
-     * @return {@link RetryResult} to indicate if the exception should be ignored
-     *     ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}),
-     *     or evaluation should proceed ({@code null}).
+     * @return {@link RetryResult} to indicate if the exception should be ignored (
+     *         {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation
+     *         should proceed ({@code null}).
      */
     RetryResult beforeEval(Exception exception);
 
@@ -78,9 +77,9 @@ boolean booleanValue() {
      *
      * @param exception the exception that is being evaluated
      * @param retryResult the result of the evaluation so far.
-     * @return {@link RetryResult} to indicate if the exception should be ignored
-     *     ({@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}),
-     *     or evaluation should proceed ({@code null}).
+     * @return {@link RetryResult} to indicate if the exception should be ignored (
+     *         {@link RetryResult#RETRY}), propagated ({@link RetryResult#ABORT}), or evaluation
+     *         should proceed ({@code null}).
      */
     RetryResult afterEval(Exception exception, RetryResult retryResult);
   }
@@ -96,14 +95,12 @@ public static class Builder {
     private final ImmutableSet.Builder> nonRetriableExceptions =
         ImmutableSet.builder();
 
-    private Builder() {
-    }
+    private Builder() {}
 
 
     /**
-     * Adds the exception handler interceptors.
-     * Call order will be maintained.
-
+     * Adds the exception handler interceptors. Call order will be maintained.
+     *
      * @param interceptors the interceptors for this exception handler
      * @return the Builder for chaining
      */
@@ -214,7 +211,7 @@ private static RetryInfo findMostSpecificRetryInfo(Set retryInfo,
       Class exception) {
     for (RetryInfo current : retryInfo) {
       if (current.exception.isAssignableFrom(exception)) {
-        RetryInfo  match = findMostSpecificRetryInfo(current.children, exception);
+        RetryInfo match = findMostSpecificRetryInfo(current.children, exception);
         return match == null ? current : match;
       }
     }
@@ -239,8 +236,8 @@ void verifyCaller(Callable callable) {
     for (Class exceptionOrError : callMethod.getExceptionTypes()) {
       Preconditions.checkArgument(Exception.class.isAssignableFrom(exceptionOrError),
           "Callable method exceptions must be derived from Exception");
-      @SuppressWarnings("unchecked") Class exception =
-          (Class) exceptionOrError;
+      @SuppressWarnings("unchecked")
+      Class exception = (Class) exceptionOrError;
       Preconditions.checkArgument(findMostSpecificRetryInfo(retryInfo, exception) != null,
           "Declared exception '" + exception + "' is not covered by exception handler");
     }
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/src/main/java/com/google/gcloud/RetryHelper.java
index 7e354a4145d3..7b47209cd3ff 100644
--- a/src/main/java/com/google/gcloud/RetryHelper.java
+++ b/src/main/java/com/google/gcloud/RetryHelper.java
@@ -35,9 +35,9 @@
 import java.util.logging.Logger;
 
 /**
- * Utility class for retrying operations.
- * For more details about the parameters, see {@link RetryParams}.
- * If the request is never successful, a {@link RetriesExhaustedException} will be thrown.
+ * Utility class for retrying operations. For more details about the parameters, see
+ * {@link RetryParams}. If the request is never successful, a {@link RetriesExhaustedException} will
+ * be thrown.
  *
  * @param  return value of the closure that is being run with retries
  */
@@ -201,8 +201,8 @@ private V doRetry() throws RetryHelperException {
       }
       long sleepDurationMillis = getSleepDuration(params, attemptNumber);
       if (log.isLoggable(Level.FINE)) {
-        log.fine(this + ": Attempt #" + attemptNumber + " failed [" + exception + "], sleeping for "
-            + sleepDurationMillis + " ms");
+        log.fine(this + ": Attempt #" + attemptNumber + " failed [" + exception
+            + "], sleeping for " + sleepDurationMillis + " ms");
       }
       try {
         Thread.sleep(sleepDurationMillis);
diff --git a/src/main/java/com/google/gcloud/RetryParams.java b/src/main/java/com/google/gcloud/RetryParams.java
index c7f25926eb9f..0d38aea77ab0 100644
--- a/src/main/java/com/google/gcloud/RetryParams.java
+++ b/src/main/java/com/google/gcloud/RetryParams.java
@@ -25,10 +25,10 @@
 import java.util.Objects;
 
 /**
- * Parameters for configuring retries with an exponential backoff.
- * Initial request is executed immediately. If the request fails but passes the
- * {@link ExceptionHandler} criteria the calling thread sleeps for {@code initialRetryDelayMillis}.
- * Each subsequent failure the sleep interval is calculated as:
+ * Parameters for configuring retries with an exponential backoff. Initial request is executed
+ * immediately. If the request fails but passes the {@link ExceptionHandler} criteria the calling
+ * thread sleeps for {@code initialRetryDelayMillis}. Each subsequent failure the sleep interval is
+ * calculated as:
  * 
    
  * {@code retryDelayBackoffFactor ^ attempts * initialRetryDelayMillis} but would be upper-bounded
  * to {@code maxRetryDelayMillis}
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index bbc8ef935a6b..36cdf64aa7f8 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -146,7 +146,7 @@ protected static String googleCloudProjectId() {
       URLConnection connection = url.openConnection();
       connection.setRequestProperty("X-Google-Metadata-Request", "True");
       try (BufferedReader reader =
-               new BufferedReader(new InputStreamReader(connection.getInputStream(), UTF_8))) {
+          new BufferedReader(new InputStreamReader(connection.getInputStream(), UTF_8))) {
         return reader.readLine();
       }
     } catch (IOException ignore) {
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 2fbaee752373..f65c8cdd02c2 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -18,6 +18,7 @@
 
 import com.google.gcloud.storage.*;
 
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -31,12 +32,14 @@
  * 
    
  * This example demonstrates a simple/typical usage.
  * 
    
- * Steps needed for running the example:
      
+ * Steps needed for running the example:
+ * 
        
  * 
      1. login using gcloud SDK - {@code gcloud auth login}.
        2. 
  * 
      2. compile using maven - {@code mvn compile}
        3. 
- * 
      3. run using maven - {@code mvn exec:java
- * -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
- * -Dexec.args="project_id list []|info [ []]|get  |upload   []|delete  "}
        4. 
+ * 
      4. run using maven -
+ * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
+ * -Dexec.args="project_id list []|info [ []]|get  |upload   []|delete  "}
+ * 
        5. 
  * 
      
  */
 public class StorageExample {
@@ -123,11 +126,6 @@ private static class DeleteAction extends BlobAction {
     public void run(StorageService storage, Blob blob) {
       storage.delete(blob);
     }
-
-    @Override
-    public String params() {
-      return " ";
-    }
   }
 
   private static class ListAction extends StorageAction {
@@ -168,7 +166,7 @@ private static class UploadAction extends StorageAction> {
     @Override
     public void run(StorageService storage, Tuple tuple) throws Exception {
       if (Files.size(tuple.first()) > 1_000_000) {
-        // todo upload via streaming API
+        // todo: upload via streaming API
         throw new IllegalArgumentException("file is too big");
       } else {
         byte[] bytes = Files.readAllBytes(tuple.first());
@@ -192,12 +190,28 @@ public String params() {
     }
   }
 
+  private static class GetAction extends BlobAction {
+    @Override
+    public void run(StorageService storage, Blob blob) {
+      blob = storage.get(blob);
+      if (blob == null) {
+        System.out.println("No such object");
+      }
+      if (blob.size() < 1_000_000) {
+        System.out.println(new String(storage.load(blob), StandardCharsets.UTF_8));
+      } else {
+        // todo: download via streaming API
+        throw new IllegalArgumentException("file is too big");
+      }
+    }
+  }
+
   static {
     ACTIONS.put("info", new InfoAction());
     ACTIONS.put("delete", new DeleteAction());
     ACTIONS.put("list", new ListAction());
     ACTIONS.put("upload", new UploadAction());
-    // todo: implement get
+    ACTIONS.put("get", new GetAction());
   }
 
   public static void printUsage() {
@@ -212,8 +226,7 @@ public static void printUsage() {
       actionAndParams.append('|');
     }
     actionAndParams.setLength(actionAndParams.length() - 1);
-    System.out.printf("Usage: %s [%s]%n",
-        StorageExample.class.getSimpleName(), actionAndParams);
+    System.out.printf("Usage: %s [%s]%n", StorageExample.class.getSimpleName(), actionAndParams);
   }
 
   @SuppressWarnings("unchecked")
@@ -232,7 +245,7 @@ public static void main(String... args) throws Exception {
     }
     StorageServiceOptions options = StorageServiceOptions.builder().project(args[0]).build();
     StorageService storage = StorageServiceFactory.instance().get(options);
-    args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length): new String []{};
+    args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length) : new String[] {};
     Object request;
     try {
       request = action.parse(args);
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 33f3d33924ab..41d4359c73d4 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.*;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      // todo: based on code consider if error is retryable
      translated = new StorageServiceException(details.getCode(), details.getMessage(), false);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Option... options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content, Option... options)
      throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Option... options) {
    try {
      return storage.buckets().get(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Option... options) {
    try {
      return storage.objects().get(object.getBucket(), object.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Option... options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Option... options) {
    try {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
    throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Option... options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Option... options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(destination.getBucket(), destination.getName(), null).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException {
    try {
      return storage.objects()
          .copy(from.getBucket(), from.getName(), to.getBucket(), to.getName(), to).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Option... options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      // todo: based on code consider if error is retryable
      translated = new StorageServiceException(details.getCode(), details.getMessage(), false);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects().get(object.getBucket(), object.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index d051438244d2..17646392686f 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.Option;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.List;

public interface StorageRpc {

  Bucket create(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject create(StorageObject object, byte[] content, Option... options)
      throws StorageServiceException;
  Iterator list() throws StorageServiceException;
  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws StorageServiceException;
  Bucket get(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject get(StorageObject object, Option... options) throws StorageServiceException;
  Bucket patch(Bucket bucket, Option... options) throws StorageServiceException;
  StorageObject patch(StorageObject storageObject, Option... options)
      throws StorageServiceException;
  void delete(Bucket bucket, Option... options) throws  StorageServiceException;
  void delete(StorageObject object, Option... options) throws StorageServiceException;
  StorageObject compose(Iterable src, StorageObject destination,
      List destinationOptions) throws StorageServiceException;
  StorageObject copy(StorageObject from, List blobSourceOptions,
      StorageObject to, List blobTargetOptions) throws StorageServiceException;
  BlobReadChannel reader(StorageObject from, Option... options) throws StorageServiceException;
  BlobWriteChannel writer(StorageObject to, Option... options) throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationMatch");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Iterator list() throws StorageServiceException;

  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  void delete(Bucket bucket, Map options) throws StorageServiceException;

  void delete(StorageObject object, Map options) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java b/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
index e12f036a3fe3..75b618f607ae 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index cb79e158d5f0..3058386d7201 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -29,9 +29,7 @@ public final class Acl implements Serializable {
   private final Role role;
 
   public enum Role {
-    OWNER,
-    READER,
-    WRITER
+    OWNER, READER, WRITER
   }
 
   public static abstract class Entity implements Serializable {
@@ -41,11 +39,7 @@ public static abstract class Entity implements Serializable {
     private final String value;
 
     public enum Type {
-      DOMAIN,
-      GROUP,
-      USER,
-      PROJECT,
-      UNKNOWN
+      DOMAIN, GROUP, USER, PROJECT, UNKNOWN
     }
 
     Entity(Type type, String value) {
@@ -162,9 +156,7 @@ public static class Project extends Entity {
     private final String projectId;
 
     enum ProjectRole {
-      OWNERS,
-      EDITORS,
-      VIEWERS
+      OWNERS, EDITORS, VIEWERS
     }
 
     Project(ProjectRole pRole, String projectId) {
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index e6b6b150ced4..6cb8e0eb8ae6 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -39,16 +39,18 @@ public class Blob implements Serializable {
 
   static final Function FROM_PB_FUNCTION =
       new Function() {
-        @Override public Blob apply(StorageObject pb) {
+        @Override
+        public Blob apply(StorageObject pb) {
           return Blob.fromPb(pb);
         }
       };
 
   static final Function TO_PB_FUNCTION = new Function() {
-        @Override public StorageObject apply(Blob blob) {
-          return blob.toPb();
-        }
-      };
+    @Override
+    public StorageObject apply(Blob blob) {
+      return blob.toPb();
+    }
+  };
 
   private final String bucket;
   private final String id;
@@ -98,8 +100,7 @@ public static final class Builder {
     private Long deleteTime;
     private Long updateTime;
 
-    private Builder() {
-    }
+    private Builder() {}
 
     public Builder bucket(String bucket) {
       this.bucket = checkNotNull(bucket);
@@ -366,7 +367,7 @@ public static Blob of(String bucket, String name) {
     return builder(bucket, name).build();
   }
 
-    public static Builder builder(Bucket bucket, String name) {
+  public static Builder builder(Bucket bucket, String name) {
     return builder(bucket.name(), name);
   }
 
@@ -378,7 +379,8 @@ StorageObject toPb() {
     StorageObject storageObject = new StorageObject();
     if (acl != null) {
       storageObject.setAcl(Lists.transform(acl, new Function() {
-        @Override public ObjectAccessControl apply(Acl acl) {
+        @Override
+        public ObjectAccessControl apply(Acl acl) {
           return acl.toObjectPb();
         }
       }));
@@ -425,7 +427,6 @@ static Blob fromPb(StorageObject storageObject) {
         .generation(storageObject.getGeneration())
         .md5(storageObject.getMd5Hash())
         .mediaLink(storageObject.getMediaLink())
-        .metadata(storageObject.getMetadata())
         .metageneration(storageObject.getMetageneration())
         .name(storageObject.getName())
         .contentDisposition(storageObject.getContentDisposition())
@@ -434,6 +435,9 @@ static Blob fromPb(StorageObject storageObject) {
         .etag(storageObject.getEtag())
         .id(storageObject.getId())
         .selfLink(storageObject.getSelfLink());
+    if (storageObject.getMetadata() != null) {
+      builder.metadata(storageObject.getMetadata());
+    }
     if (storageObject.getTimeDeleted() != null) {
       builder.deleteTime(storageObject.getTimeDeleted().getValue());
     }
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
index 3757dec04e0b..32b68b3e97f6 100644
--- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
@@ -23,9 +23,9 @@
 /**
  * A channel for reading data from a Google Cloud Storage object.
  *
- *  Implementations of this class may buffer data internally to reduce remote calls.
+ * Implementations of this class may buffer data internally to reduce remote calls.
  *
- *  This class is @{link Serializable}, which allows incremental reads.
+ * This class is @{link Serializable}, which allows incremental reads.
  */
 public interface BlobReadChannel extends ReadableByteChannel, Serializable, Closeable {
 
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
index a83ed661baca..77ce84a8ea7a 100644
--- a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
@@ -23,9 +23,9 @@
 /**
  * A channel for writing data to a Google Cloud Storage object.
  *
- * Implementations of this class may further buffer data internally to reduce remote calls.
- * Written data will only be visible after calling {@link #close()}.
- * This class is serializable, to allow incremental writes.
+ * Implementations of this class may further buffer data internally to reduce remote calls. Written
+ * data will only be visible after calling {@link #close()}. This class is serializable, to allow
+ * incremental writes.
  */
 public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable {
 
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 1c3077e41bb6..16937fda2b4b 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -60,14 +60,16 @@ public final class Bucket implements Serializable {
 
   static final Function FROM_PB_FUNCTION =
       new Function() {
-        @Override public Bucket apply(com.google.api.services.storage.model.Bucket pb) {
+        @Override
+        public Bucket apply(com.google.api.services.storage.model.Bucket pb) {
           return Bucket.fromPb(pb);
         }
       };
 
   static final Function TO_PB_FUNCTION =
       new Function() {
-        @Override public com.google.api.services.storage.model.Bucket apply(Bucket bucket) {
+        @Override
+        public com.google.api.services.storage.model.Bucket apply(Bucket bucket) {
           return bucket.toPb();
         }
       };
@@ -226,8 +228,7 @@ public static final class StorageClass implements Serializable {
     private final String value;
 
     public enum Option {
-      DURABLE_REDUCED_AVAILABILITY,
-      STANDARD;
+      DURABLE_REDUCED_AVAILABILITY, STANDARD;
 
       private final StorageClass storageClass;
 
@@ -345,8 +346,7 @@ public final static class Builder {
     private ImmutableList acl;
     private ImmutableList defaultAcl;
 
-    private Builder() {
-    }
+    private Builder() {}
 
     public Builder name(String name) {
       this.name = checkNotNull(name);
@@ -573,14 +573,16 @@ com.google.api.services.storage.model.Bucket toPb() {
     }
     if (acl != null) {
       bucketPb.setAcl(transform(acl, new Function() {
-        @Override public BucketAccessControl apply(Acl acl) {
+        @Override
+        public BucketAccessControl apply(Acl acl) {
           return acl.toBucketPb();
         }
       }));
     }
     if (defaultAcl != null) {
       bucketPb.setDefaultObjectAcl(transform(defaultAcl, new Function() {
-        @Override public ObjectAccessControl apply(Acl acl) {
+        @Override
+        public ObjectAccessControl apply(Acl acl) {
           return acl.toObjectPb();
         }
       }));
@@ -601,7 +603,8 @@ com.google.api.services.storage.model.Bucket toPb() {
     if (deleteRules != null) {
       Lifecycle lifecycle = new Lifecycle();
       lifecycle.setRule(transform(deleteRules, new Function() {
-        @Override public Rule apply(DeleteRule deleteRule) {
+        @Override
+        public Rule apply(DeleteRule deleteRule) {
           return deleteRule.toPb();
         }
       }));
@@ -626,7 +629,8 @@ static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) {
     }
     if (bucketPb.getAcl() != null) {
       builder.acl(transform(bucketPb.getAcl(), new Function() {
-        @Override public Acl apply(BucketAccessControl bucketAccessControl) {
+        @Override
+        public Acl apply(BucketAccessControl bucketAccessControl) {
           return Acl.fromPb(bucketAccessControl);
         }
       }));
@@ -654,7 +658,8 @@ public Acl apply(ObjectAccessControl objectAccessControl) {
     if (bucketPb.getLifecycle() != null && bucketPb.getLifecycle().getRule() != null) {
       builder.deleteRules(transform(bucketPb.getLifecycle().getRule(),
           new Function() {
-            @Override public DeleteRule apply(Rule rule) {
+            @Override
+            public DeleteRule apply(Rule rule) {
               return DeleteRule.fromPb(rule);
             }
           }));
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index 87d33348983d..ccbe2413042e 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -35,13 +35,15 @@ public final class Cors implements Serializable {
   private static final long serialVersionUID = -8637770919343335655L;
 
   static final Function FROM_PB_FUNCTION = new Function() {
-    @Override public Cors apply(Bucket.Cors pb) {
+    @Override
+    public Cors apply(Bucket.Cors pb) {
       return Cors.fromPb(pb);
     }
   };
 
   static final Function TO_PB_FUNCTION = new Function() {
-    @Override public Bucket.Cors apply(Cors cors) {
+    @Override
+    public Bucket.Cors apply(Cors cors) {
       return cors.toPb();
     }
   };
@@ -103,8 +105,7 @@ public static final class Builder {
     private ImmutableList origins;
     private ImmutableList responseHeaders;
 
-    private Builder() {
-    }
+    private Builder() {}
 
     public Builder maxAgeSeconds(Integer maxAgeSeconds) {
       this.maxAgeSeconds = maxAgeSeconds;
@@ -178,12 +179,14 @@ Bucket.Cors toPb() {
   static Cors fromPb(Bucket.Cors cors) {
     Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds());
     builder.methods(transform(cors.getMethod(), new Function() {
-      @Override public Method apply(String name) {
+      @Override
+      public Method apply(String name) {
         return Method.valueOf(name.toUpperCase());
       }
     }));
     builder.origins(transform(cors.getOrigin(), new Function() {
-      @Override public Origin apply(String value) {
+      @Override
+      public Origin apply(String value) {
         return Origin.of(value);
       }
     }));
diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java
index 05257babca00..d571b176d9ab 100644
--- a/src/main/java/com/google/gcloud/storage/ListOptions.java
+++ b/src/main/java/com/google/gcloud/storage/ListOptions.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    private Builder() {
    }

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }

    public ListOptions build() {
      return new ListOptions(this);
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }

  public static ListOptions of() {
    return builder().build();
  }

  public static Builder builder() {
    return new Builder();
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    private Builder() {}

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }

    public ListOptions build() {
      return new ListOptions(this);
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }

  public static ListOptions of() {
    return builder().build();
  }

  public static Builder builder() {
    return new Builder();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Option.java b/src/main/java/com/google/gcloud/storage/Option.java
index cdbf00015b72..7e130603b79c 100644
--- a/src/main/java/com/google/gcloud/storage/Option.java
+++ b/src/main/java/com/google/gcloud/storage/Option.java
@@ -18,29 +18,32 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.common.base.MoreObjects;
+import com.google.gcloud.spi.StorageRpc;
+
 import java.io.Serializable;
 import java.util.Objects;
 
 /**
  * Base class for Storage operation option
  */
-public class Option implements Serializable {
+class Option implements Serializable {
 
   private static final long serialVersionUID = -73199088766477208L;
 
-  private final String name;
+  private final StorageRpc.Option rpcOption;
   private final Object value;
 
-  Option(String name, Object value) {
-    this.name = checkNotNull(name);
-    this.value = value;
+  Option(StorageRpc.Option rpcOption, Object value) {
+    this.rpcOption = checkNotNull(rpcOption);
+    this.value = checkNotNull(value);
   }
 
-  public String name() {
-    return name;
+  StorageRpc.Option rpcOption() {
+    return rpcOption;
   }
 
-  public Object value() {
+  Object value() {
     return value;
   }
 
@@ -50,12 +53,20 @@ public boolean equals(Object obj) {
       return false;
     }
     Option other = (Option) obj;
-    return Objects.equals(name, other.name)
+    return Objects.equals(rpcOption, other.rpcOption)
         && Objects.equals(value, other.value);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(name, value);
+    return Objects.hash(rpcOption, value);
+  }
+
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(this)
+        .add("name", rpcOption.value())
+        .add("value", value)
+        .toString();
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index ca3752e08b28..2fb92cd66743 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -21,6 +21,7 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.gcloud.Service;
+import com.google.gcloud.spi.StorageRpc;
 
 import java.io.Serializable;
 import java.util.Arrays;
@@ -33,7 +34,8 @@
 
 public interface StorageService extends Service {
 
-  // todo: provide way for construct signed URLs - https://cloud.google.com/storage/docs/access-control#Signed-URLs
+  // todo: provide way for construct signed URLs -
+  // https://cloud.google.com/storage/docs/access-control#Signed-URLs
 
   enum PredefinedAcl {
     AUTHENTICATED_READ("authenticatedRead"),
@@ -60,20 +62,20 @@ class BucketTargetOption extends Option {
 
     private static final long serialVersionUID = -5880204616982900975L;
 
-    private BucketTargetOption(String name, Object value) {
-      super(name, value);
+    private BucketTargetOption(StorageRpc.Option rpcOption, Object value) {
+      super(rpcOption, value);
     }
 
     public static BucketTargetOption predefinedAcl(PredefinedAcl acl) {
-      return new BucketTargetOption("predefinedAcl", acl.entry());
+      return new BucketTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
 
     public static BucketTargetOption predefinedDefaultObjectAcl(PredefinedAcl acl) {
-      return new BucketTargetOption("predefinedDefaultObjectAcl", acl.entry());
+      return new BucketTargetOption(StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL, acl.entry());
     }
 
     public static BucketTargetOption metagenerationMatch(boolean match) {
-      return new BucketTargetOption("ifMetagenerationMatch", match);
+      return new BucketTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
     }
   }
 
@@ -81,12 +83,12 @@ class BucketSourceOption extends Option {
 
     private static final long serialVersionUID = 5185657617120212117L;
 
-    private BucketSourceOption(String name, Object value) {
-      super(name, value);
+    private BucketSourceOption(StorageRpc.Option rpcOption, Object value) {
+      super(rpcOption, value);
     }
 
     public static BucketSourceOption metagenerationMatch(boolean match) {
-      return new BucketSourceOption("ifMetagenerationMatch", match);
+      return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
     }
   }
 
@@ -94,20 +96,20 @@ class BlobTargetOption extends Option {
 
     private static final long serialVersionUID = 214616862061934846L;
 
-    private BlobTargetOption(String name, Object value) {
-      super(name, value);
+    private BlobTargetOption(StorageRpc.Option rpcOption, Object value) {
+      super(rpcOption, value);
     }
 
     public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
-      return new BlobTargetOption("predefinedAcl", acl.entry());
+      return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
 
     public static BlobTargetOption generationMath(boolean match) {
-      return new BlobTargetOption("ifGenerationMatch", match);
+      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, match);
     }
 
     public static BlobTargetOption metagenerationMatch(boolean match) {
-      return new BlobTargetOption("ifMetagenerationMatch", match);
+      return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
     }
   }
 
@@ -115,16 +117,16 @@ class BlobSourceOption extends Option {
 
     private static final long serialVersionUID = -3712768261070182991L;
 
-    private BlobSourceOption(String name, Object value) {
-      super(name, value);
+    private BlobSourceOption(StorageRpc.Option rpcOption, Object value) {
+      super(rpcOption, value);
     }
 
     public static BlobSourceOption generationMath(boolean match) {
-      return new BlobSourceOption("ifGenerationMatch", match);
+      return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH, match);
     }
 
     public static BlobSourceOption metagenerationMatch(boolean match) {
-      return new BlobSourceOption("ifMetagenerationMatch", match);
+      return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
     }
   }
 
@@ -303,12 +305,12 @@ public static Builder builder() {
   /**
    * @throws StorageServiceException upon failure
    */
-  Bucket create(Bucket bucket, BucketTargetOption... option);
+  Bucket create(Bucket bucket, BucketTargetOption... options);
 
   /**
    * @throws StorageServiceException upon failure
    */
-  Blob create(Blob blob, byte[] content, BlobTargetOption... option);
+  Blob create(Blob blob, byte[] content, BlobTargetOption... options);
 
   /**
    * @throws StorageServiceException upon failure
@@ -333,22 +335,22 @@ public static Builder builder() {
   /**
    * @throws StorageServiceException upon failure
    */
-  Bucket update(Bucket bucket, BucketTargetOption... option);
+  Bucket update(Bucket bucket, BucketTargetOption... options);
 
   /**
    * @throws StorageServiceException upon failure
    */
-  Blob update(Blob blob, BlobTargetOption... option);
+  Blob update(Blob blob, BlobTargetOption... options);
 
   /**
    * @throws StorageServiceException upon failure
    */
-  void delete(Bucket bucket, BucketSourceOption... option);
+  void delete(Bucket bucket, BucketSourceOption... options);
 
   /**
    * @throws StorageServiceException upon failure
    */
-  void delete(Blob blob, BlobSourceOption... option);
+  void delete(Blob blob, BlobSourceOption... options);
 
   /**
    * @throws StorageServiceException upon failure
@@ -360,13 +362,19 @@ public static Builder builder() {
    */
   Blob copy(CopyRequest copyRequest);
 
+
+  /**
+   * @throws StorageServiceException upon failure
+   */
+  byte[] load(Blob blob, BlobSourceOption... options);
+
   /**
    * @throws StorageServiceException upon failure
    */
-  BlobReadChannel readFrom(Blob blob, BlobSourceOption... option);
+  BlobReadChannel reader(Blob blob, BlobSourceOption... options);
 
   /**
    * @throws StorageServiceException upon failure
    */
-  BlobWriteChannel writeTo(Blob blob, BlobTargetOption... option);
+  BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
index 4a3438819250..900a762d46d2 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceException.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceException.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

/**
 * Storage service exception.
 *
 * @see Google Cloud Storage error codes
 */
public class StorageServiceException extends RuntimeException {

  private static final long serialVersionUID = -3748432005065428084L;

  private final int code;
  private final boolean retryable;

  public StorageServiceException(int code, String message, boolean retryable) {
    super(message);
    this.code = code;
    this.retryable = retryable;
  }

  /**
   * Returns the code associated with this exception.
   */
  public int code() {
    return code;
  }

  public boolean retryable() {
    return retryable;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

/**
 * Storage service exception.
 *
 * @see Google Cloud
 *      Storage error codes
 */
public class StorageServiceException extends RuntimeException {

  private static final long serialVersionUID = -3748432005065428084L;

  private final int code;
  private final boolean retryable;

  public StorageServiceException(int code, String message, boolean retryable) {
    super(message);
    this.code = code;
    this.retryable = retryable;
  }

  /**
   * Returns the code associated with this exception.
   */
  public int code() {
    return code;
  }

  public boolean retryable() {
    return retryable;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
index 2c2053b72d6b..a801622082ef 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
@@ -21,10 +21,11 @@
 public abstract class StorageServiceFactory {
 
   private static final StorageServiceFactory INSTANCE = new StorageServiceFactory() {
-      @Override public StorageService get(StorageServiceOptions options) {
-        return new StorageServiceImpl(options);
-      }
-    };
+    @Override
+    public StorageService get(StorageServiceOptions options) {
+      return new StorageServiceImpl(options);
+    }
+  };
 
   public static StorageServiceFactory instance() {
     return INSTANCE;
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 7d653a7fb25a..afaca940c9f0 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -20,6 +20,7 @@
 
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.gcloud.BaseService;
@@ -28,34 +29,34 @@
 import com.google.gcloud.RetryParams;
 import com.google.gcloud.spi.StorageRpc;
 
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.Callable;
 
 final class StorageServiceImpl extends BaseService implements StorageService {
 
-  private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
-      new Interceptor() {
+  private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR = new Interceptor() {
 
-        private static final long serialVersionUID = -7758580330857881124L;
+    private static final long serialVersionUID = -7758580330857881124L;
 
-        @Override
-        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
-          return null;
-        }
+    @Override
+    public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+      return null;
+    }
 
-        @Override
-        public RetryResult beforeEval(Exception exception) {
-          if (exception instanceof StorageServiceException) {
-            boolean retriable = ((StorageServiceException) exception).retryable();
-            return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
-          }
-          return null;
-        }
-      };
+    @Override
+    public RetryResult beforeEval(Exception exception) {
+      if (exception instanceof StorageServiceException) {
+        boolean retriable = ((StorageServiceException) exception).retryable();
+        return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
+      }
+      return null;
+    }
+  };
   private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
-      .abortOn(RuntimeException.class)
-      .interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
+      .abortOn(RuntimeException.class).interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
 
   private final StorageRpc storageRpc;
   private final RetryParams retryParams;
@@ -68,56 +69,68 @@ public RetryResult beforeEval(Exception exception) {
   }
 
   @Override
-  public Bucket create(Bucket bucket, final BucketTargetOption... options) {
+  public Bucket create(Bucket bucket, BucketTargetOption... options) {
+    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
+    final Map optionsMap = optionMap(options);
     return Bucket.fromPb(runWithRetries(
         new Callable() {
-          @Override public com.google.api.services.storage.model.Bucket call() {
-            return storageRpc.create(bucketPb, options);
+          @Override
+          public com.google.api.services.storage.model.Bucket call() {
+            return storageRpc.create(bucketPb, optionsMap);
           }
         }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob create(Blob blob, final byte[] content, final BlobTargetOption... options) {
+  public Blob create(Blob blob, final byte[] content, BlobTargetOption... options) {
+    Validator.checkBlobOptions(blob, options);
     final StorageObject blobPb = blob.toPb();
+    final Map optionsMap = optionMap(options);
     return Blob.fromPb(runWithRetries(new Callable() {
-          @Override public StorageObject call() {
-            return storageRpc.create(blobPb, content, options);
-          }
-        }, retryParams, EXCEPTION_HANDLER));
+      @Override
+      public StorageObject call() {
+        return storageRpc.create(blobPb, content, optionsMap);
+      }
+    }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Bucket get(Bucket bucket, final BucketSourceOption... options) {
+  public Bucket get(Bucket bucket, BucketSourceOption... options) {
+    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
+    final Map optionsMap = optionMap(options);
     return Bucket.fromPb(runWithRetries(
         new Callable() {
-          @Override public com.google.api.services.storage.model.Bucket call() {
-            return storageRpc.get(bucketPb, options);
+          @Override
+          public com.google.api.services.storage.model.Bucket call() {
+            return storageRpc.get(bucketPb, optionsMap);
           }
         }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob get(Blob blob, final BlobSourceOption... options) {
+  public Blob get(Blob blob, BlobSourceOption... options) {
+    Validator.checkBlobOptions(blob, options);
     final StorageObject storedObject = blob.toPb();
+    final Map optionsMap = optionMap(options);
     return Blob.fromPb(runWithRetries(new Callable() {
-      @Override public StorageObject call() {
-        return storageRpc.get(storedObject, options);
+      @Override
+      public StorageObject call() {
+        return storageRpc.get(storedObject, optionsMap);
       }
     }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
   public Iterator list() {
-    return Iterators.transform(runWithRetries(
-        new Callable>() {
-          @Override public Iterator call() {
+    return Iterators.transform(
+        runWithRetries(new Callable>() {
+          @Override
+          public Iterator call() {
             return storageRpc.list();
           }
-        }, retryParams, EXCEPTION_HANDLER),
-        Bucket.FROM_PB_FUNCTION);
+        }, retryParams, EXCEPTION_HANDLER), Bucket.FROM_PB_FUNCTION);
   }
 
   @Override
@@ -126,48 +139,59 @@ public Iterator list(String bucket, ListOptions settings) {
     String delimiter = settings.recursive() ? options().pathDelimiter() : null;
     return Iterators.transform(
         storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(),
-            settings.includeOlderVersions(), settings.maxResults()),
-        Blob.FROM_PB_FUNCTION);
+            settings.includeOlderVersions(), settings.maxResults()), Blob.FROM_PB_FUNCTION);
   }
 
   @Override
-  public Bucket update(Bucket bucket, final BucketTargetOption... options) {
+  public Bucket update(Bucket bucket, BucketTargetOption... options) {
+    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
+    final Map optionsMap = optionMap(options);
     return Bucket.fromPb(runWithRetries(
         new Callable() {
-          @Override public com.google.api.services.storage.model.Bucket call() {
-            return storageRpc.patch(bucketPb, options);
+          @Override
+          public com.google.api.services.storage.model.Bucket call() {
+            return storageRpc.patch(bucketPb, optionsMap);
           }
         }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob update(Blob blob, final BlobTargetOption... options) {
+  public Blob update(Blob blob, BlobTargetOption... options) {
+    Validator.checkBlobOptions(blob, options);
     final StorageObject storageObject = blob.toPb();
+    final Map optionsMap = optionMap(options);
     return Blob.fromPb(runWithRetries(new Callable() {
-      @Override public StorageObject call() {
-        return storageRpc.patch(storageObject, options);
+      @Override
+      public StorageObject call() {
+        return storageRpc.patch(storageObject, optionsMap);
       }
     }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public void delete(Bucket bucket, final BucketSourceOption... options) {
+  public void delete(Bucket bucket, BucketSourceOption... options) {
+    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
+    final Map optionsMap = optionMap(options);
     runWithRetries(new Callable() {
-      @Override public Void call() {
-        storageRpc.delete(bucketPb, options);
+      @Override
+      public Void call() {
+        storageRpc.delete(bucketPb, optionsMap);
         return null;
       }
     }, retryParams, EXCEPTION_HANDLER);
   }
 
   @Override
-  public void delete(Blob blob, final BlobSourceOption... options) {
+  public void delete(Blob blob, BlobSourceOption... options) {
+    Validator.checkBlobOptions(blob, options);
     final StorageObject storageObject = blob.toPb();
+    final Map optionsMap = optionMap(options);
     runWithRetries(new Callable() {
-      @Override public Void call() {
-        storageRpc.delete(storageObject, options);
+      @Override
+      public Void call() {
+        storageRpc.delete(storageObject, optionsMap);
         return null;
       }
     }, retryParams, EXCEPTION_HANDLER);
@@ -179,39 +203,70 @@ public Blob compose(final ComposeRequest composeRequest) {
         Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size());
     for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) {
       sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob)
-          .generation(sourceBlob.generation)
-          .build()
-          .toPb());
+          .generation(sourceBlob.generation).build().toPb());
     }
     final StorageObject target = composeRequest.target().toPb();
+    final Map targetOptions = optionMap(composeRequest.targetOptions());
     return Blob.fromPb(runWithRetries(new Callable() {
-      @Override public StorageObject call() {
-        return storageRpc.compose(sources, target, composeRequest.targetOptions());
+      @Override
+      public StorageObject call() {
+        return storageRpc.compose(sources, target, targetOptions);
       }
     }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public Blob copy(final CopyRequest copyRequest) {
+  public Blob copy(CopyRequest copyRequest) {
     final StorageObject source = copyRequest.source().toPb();
+    final Map sourceOptions = optionMap(copyRequest.sourceOptions());
     final StorageObject target = copyRequest.target().toPb();
+    final Map targetOptions = optionMap(copyRequest.targetOptions());
     return Blob.fromPb(runWithRetries(new Callable() {
-      @Override public StorageObject call() {
-        return storageRpc.copy(source, copyRequest.sourceOptions(), target,
-            copyRequest.targetOptions());
+      @Override
+      public StorageObject call() {
+        return storageRpc.copy(source, sourceOptions, target, targetOptions);
       }
     }, retryParams, EXCEPTION_HANDLER));
   }
 
   @Override
-  public BlobReadChannel readFrom(Blob blob, BlobSourceOption... options) {
+  public byte[] load(Blob blob, BlobSourceOption... options) {
+    Validator.checkBlobOptions(blob, options);
+    final StorageObject storageObject = blob.toPb();
+    final Map optionsMap = optionMap(options);
+    return runWithRetries(new Callable() {
+      @Override
+      public byte[] call() {
+        return storageRpc.load(storageObject, optionsMap);
+      }
+    }, retryParams, EXCEPTION_HANDLER);
+  }
+
+  @Override
+  public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
+    Validator.checkBlobOptions(blob, options);
     // todo: Use retry helper on retriable failures
-    return storageRpc.reader(blob.toPb(), options);
+    final Map optionsMap = optionMap(options);
+    return storageRpc.reader(blob.toPb(), optionsMap);
   }
 
   @Override
-  public BlobWriteChannel writeTo(Blob blob, BlobTargetOption... options) {
+  public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
+    Validator.checkBlobOptions(blob, options);
     // todo: Use retry helper on retriable failures
-    return storageRpc.writer(blob.toPb(), options);
+    final Map optionsMap = optionMap(options);
+    return storageRpc.writer(blob.toPb(), optionsMap);
+  }
+
+  private static Map optionMap(Iterable options) {
+    ImmutableMap.Builder mapBuider = ImmutableMap.builder();
+    for (Option option : options) {
+      mapBuider.put(option.rpcOption(), option.value());
+    }
+    return mapBuider.build();
+  }
+
+  private static Map optionMap(Option... options) {
+    return optionMap(Arrays.asList(options));
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 10b54d79e413..c36d28c1293f 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -42,8 +42,7 @@ public static class Builder extends ServiceOptions.Builder {
     private String pathDelimiter;
     private StorageRpc storageRpc;
 
-    private Builder() {
-    }
+    private Builder() {}
 
     private Builder(StorageServiceOptions options) {
       super(options);
diff --git a/src/main/java/com/google/gcloud/storage/Validator.java b/src/main/java/com/google/gcloud/storage/Validator.java
index 0db8fd3be16e..5590d67528cc 100644
--- a/src/main/java/com/google/gcloud/storage/Validator.java
+++ b/src/main/java/com/google/gcloud/storage/Validator.java
@@ -18,21 +18,37 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import java.util.Arrays;
+
 /**
  * Utility to validate Storage type/values.
  */
 public class Validator {
 
+  static void checkBlobOptions(Blob blob, Option... options) {
+    checkBlobOptions("requested", blob, Arrays.asList(options));
+  }
+
   static void checkBlobOptions(String name, Blob blob, Iterable options) {
     for (Option option : options) {
-      switch (option.name()) {
-        case "ifGenerationMatch":
+      switch (option.rpcOption()) {
+        case IF_GENERATION_MATCH:
           checkArgument(blob.generation() != 0, "%s blob is missing generation", name);
           break;
-        case "ifMetagenerationMatch":
+        case IF_METAGENERATION_MATCH:
           checkArgument(blob.metageneration() != 0, "%s blob is missing metageneration", name);
           break;
       }
     }
   }
+
+  static void checkBucketOptions(Bucket bucket, Option... options) {
+    for (Option option : options) {
+      switch (option.rpcOption()) {
+        case IF_METAGENERATION_MATCH:
+          checkArgument(bucket.metageneration() != 0, "bucket is missing metageneration");
+          break;
+      }
+    }
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/src/main/java/com/google/gcloud/storage/package-info.java
index 9aabec93c93f..0aa916666e8c 100644
--- a/src/main/java/com/google/gcloud/storage/package-info.java
+++ b/src/main/java/com/google/gcloud/storage/package-info.java
@@ -20,3 +20,4 @@
  * @see Google Cloud Storageg
  */
 package com.google.gcloud.storage;
+
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
index 1d96e86d0e46..3844f9de36d7 100644
--- a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
+++ b/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
@@ -129,7 +129,8 @@ public void testShouldTry() {
     assertTrue(handler.shouldRetry(new NullPointerException()));
 
     final AtomicReference before = new AtomicReference<>(RetryResult.ABORT);
-    @SuppressWarnings("serial") Interceptor interceptor = new Interceptor() {
+    @SuppressWarnings("serial")
+    Interceptor interceptor = new Interceptor() {
 
       @Override
       public RetryResult afterEval(Exception exception, RetryResult retryResult) {

From 2217787f1d869b1e45eb57df0a3b519150373d5e Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 23 Apr 2015 17:09:36 -0700
Subject: [PATCH 178/732] make AuthCredentials serializable

---
 .../com/google/gcloud/AuthCredentials.java    | 79 ++++++++++++++-----
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 2 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java
index 3e6c3651c576..ff5d20add781 100644
--- a/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -30,6 +30,8 @@
 import com.google.auth.oauth2.GoogleCredentials;
 
 import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
 import java.security.GeneralSecurityException;
 import java.security.PrivateKey;
 import java.util.Set;
@@ -37,10 +39,14 @@
 /**
  * Credentials for accessing Google Cloud services.
  */
-public abstract class AuthCredentials {
+public abstract class AuthCredentials implements Serializable {
+
+  private static final long serialVersionUID = 236297804453464604L;
 
   private static class AppEngineAuthCredentials extends AuthCredentials {
 
+    private static final long serialVersionUID = 7931300552744202954L;
+
     @Override
     protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
         Set scopes) {
@@ -50,6 +56,7 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
 
   private static class ServiceAccountAuthCredentials extends AuthCredentials {
 
+    private static final long serialVersionUID = 8007708734318445901L;
     private final String account;
     private final PrivateKey privateKey;
 
@@ -78,6 +85,54 @@ protected HttpRequestInitializer httpRequestInitializer(
     }
   }
 
+  private static class ComputeEngineAuthCredentials extends AuthCredentials {
+
+    private static final long serialVersionUID = -5217355402127260144L;
+
+    private transient ComputeCredential computeCredential;
+
+    ComputeEngineAuthCredentials() throws IOException, GeneralSecurityException {
+      computeCredential = getComputeCredential();
+    }
+
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+      in.defaultReadObject();
+      try {
+        computeCredential = getComputeCredential();
+      } catch (GeneralSecurityException e) {
+       throw new IOException(e);
+      }
+    }
+
+    @Override
+    protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
+        Set scopes) {
+      return computeCredential;
+    }
+  }
+
+  private static class ApplicationDefaultAuthCredentials extends AuthCredentials {
+
+    private static final long serialVersionUID = -8306873864136099893L;
+    
+    private transient GoogleCredentials googleCredentials;
+
+    ApplicationDefaultAuthCredentials() throws IOException {
+      googleCredentials = GoogleCredentials.getApplicationDefault();
+    }
+
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+      in.defaultReadObject();
+      googleCredentials =  GoogleCredentials.getApplicationDefault();
+    }
+
+    @Override
+    protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
+        Set scopes) {
+      return new HttpCredentialsAdapter(googleCredentials);
+    }
+  }
+
   protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
       Set scopes);
 
@@ -85,16 +140,9 @@ public static AuthCredentials createForAppEngine() {
     return new AppEngineAuthCredentials();
   }
 
-  public static AuthCredentials createForComputeEngine() throws IOException,
-      GeneralSecurityException {
-    final ComputeCredential cred = getComputeCredential();
-    return new AuthCredentials() {
-      @Override
-      protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
-          Set scopes) {
-        return cred;
-      }
-    };
+  public static AuthCredentials createForComputeEngine()
+      throws IOException, GeneralSecurityException {
+    return new ComputeEngineAuthCredentials();
   }
 
   /**
@@ -111,14 +159,7 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
    * @throws IOException if the credentials cannot be created in the current environment.
    */
   public static AuthCredentials createApplicationDefaults() throws IOException {
-    final GoogleCredentials credentials = GoogleCredentials.getApplicationDefault();
-    return new AuthCredentials() {
-      @Override
-      protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
-          Set scopes) {
-        return new HttpCredentialsAdapter(credentials);
-      }
-    };
+    return new ApplicationDefaultAuthCredentials();
   }
 
   public static AuthCredentials createFor(String account, PrivateKey privateKey) {
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 41d4359c73d4..867913c2f286 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;

public class DefaultStorageRpc implements StorageRpc {

  private final StorageServiceOptions options;
  private final Storage storage;

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: set projection to full
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      // todo: based on code consider if error is retryable
      translated = new StorageServiceException(details.getCode(), details.getMessage(), false);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets().insert(this.options.project(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects().insert(storageObject.getBucket(), storageObject,
          new AbstractInputStreamContent(storageObject.getContentType()) {
            @Override
            public InputStream getInputStream() throws IOException {
              return new ByteArrayInputStream(content);
            }

            @Override
            public long getLength() throws IOException {
              return content.length;
            }

            @Override
            public boolean retrySupported() {
              return true;
            }
          }).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets().list(options.project()).execute().getItems().iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects().get(object.getBucket(), object.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets().patch(bucket.getName(), bucket).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    try {
      return storage.objects().compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets()
          .list(options.project())
          .setProjection(DEFAULT_PROJECTION)
          .execute()
          .getItems()
          .iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    // todo: missing setProjection
    try {
      return storage.objects()
          .compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {

    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file

From af43ec053c7c813911228957f6f3074d1a47d441 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 23 Apr 2015 17:27:16 -0700
Subject: [PATCH 179/732] work in progress

---
 .../com/google/gcloud/AuthCredentials.java    |  2 +-
 .../com/google/gcloud/ServiceOptions.java     | 68 +++++++++++--------
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 3 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java
index ff5d20add781..e6c783565d8f 100644
--- a/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -114,7 +114,7 @@ protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
   private static class ApplicationDefaultAuthCredentials extends AuthCredentials {
 
     private static final long serialVersionUID = -8306873864136099893L;
-    
+
     private transient GoogleCredentials googleCredentials;
 
     ApplicationDefaultAuthCredentials() throws IOException {
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 36cdf64aa7f8..896d5c1572f5 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -28,24 +28,54 @@
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.Serializable;
 import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.Set;
 
-public abstract class ServiceOptions {
+public abstract class ServiceOptions implements Serializable {
 
   private static final String DEFAULT_HOST = "https://www.googleapis.com";
+  private static final long serialVersionUID = 1203687993961393350L;
 
   private final String host;
-  private final HttpTransport httpTransport;
+  private final HttpTransportFactory httpTransportFactory;
   private final AuthCredentials authCredentials;
   private final RetryParams retryParams;
 
+  public interface HttpTransportFactory extends Serializable {
+    HttpTransport create();
+  }
+
+  private static class DefaultHttpTransportFactory implements HttpTransportFactory {
+
+    private static final long serialVersionUID = 2684907202489506911L;
+
+    @Override
+    public HttpTransport create() {
+      // Consider App Engine
+      if (appEngineAppId() != null) {
+        try {
+          return new UrlFetchTransport();
+        } catch (Exception ignore) {
+          // Maybe not on App Engine
+        }
+      }
+      // Consider Compute
+      try {
+        return AuthCredentials.getComputeCredential().getTransport();
+      } catch (Exception e) {
+        // Maybe not on GCE
+      }
+      return new NetHttpTransport();
+    }
+  }
+
   protected abstract static class Builder> {
 
     private String host;
-    private HttpTransport httpTransport;
+    private HttpTransportFactory httpTransportFactory;
     private AuthCredentials authCredentials;
     private RetryParams retryParams;
 
@@ -53,7 +83,7 @@ protected Builder() {}
 
     protected Builder(ServiceOptions options) {
       host = options.host;
-      httpTransport = options.httpTransport;
+      httpTransportFactory = options.httpTransportFactory;
       authCredentials = options.authCredentials;
       retryParams = options.retryParams;
     }
@@ -70,8 +100,8 @@ public B host(String host) {
       return self();
     }
 
-    public B httpTransport(HttpTransport httpTransport) {
-      this.httpTransport = httpTransport;
+    public B httpTransportFactory(HttpTransportFactory httpTransportFactory) {
+      this.httpTransportFactory = httpTransportFactory;
       return self();
     }
 
@@ -88,29 +118,12 @@ public B retryParams(RetryParams retryParams) {
 
   protected ServiceOptions(Builder builder) {
     host = firstNonNull(builder.host, DEFAULT_HOST);
-    httpTransport = firstNonNull(builder.httpTransport, defaultHttpTransport());
+    httpTransportFactory =
+        firstNonNull(builder.httpTransportFactory, new DefaultHttpTransportFactory());
     authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials());
     retryParams = builder.retryParams;
   }
 
-  private static HttpTransport defaultHttpTransport() {
-    // Consider App Engine
-    if (appEngineAppId() != null) {
-      try {
-        return new UrlFetchTransport();
-      } catch (Exception ignore) {
-        // Maybe not on App Engine
-      }
-    }
-    // Consider Compute
-    try {
-      return AuthCredentials.getComputeCredential().getTransport();
-    } catch (Exception e) {
-      // Maybe not on GCE
-    }
-    return new NetHttpTransport();
-  }
-
   private static AuthCredentials defaultAuthCredentials() {
     // Consider App Engine. This will not be needed once issue #21 is fixed.
     if (appEngineAppId() != null) {
@@ -179,8 +192,8 @@ public String host() {
     return host;
   }
 
-  public HttpTransport httpTransport() {
-    return httpTransport;
+  public HttpTransportFactory httpTransportFactory() {
+    return httpTransportFactory;
   }
 
   public AuthCredentials authCredentials() {
@@ -192,6 +205,7 @@ public RetryParams retryParams() {
   }
 
   public HttpRequestInitializer httpRequestInitializer() {
+    HttpTransport httpTransport = httpTransportFactory.create();
     return authCredentials().httpRequestInitializer(httpTransport, scopes());
   }
 
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 867913c2f286..5451375e64f8 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransport();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets()
          .list(options.project())
          .setProjection(DEFAULT_PROJECTION)
          .execute()
          .getItems()
          .iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    // todo: missing setProjection
    try {
      return storage.objects()
          .compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {

    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets()
          .list(options.project())
          .setProjection(DEFAULT_PROJECTION)
          .execute()
          .getItems()
          .iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    // todo: missing setProjection
    try {
      return storage.objects()
          .compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {

    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file

From 5707915a9809a736c5cf5e5b34906d25eb512186 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 23 Apr 2015 22:27:12 -0700
Subject: [PATCH 180/732] work in progress

---
 .../com/google/gcloud/ServiceOptions.java     | 23 +++++++++++---
 .../datastore/DatastoreServiceOptions.java    | 30 +++++++------------
 .../google/gcloud/spi/ServiceRpcFactory.java  |  2 +-
 .../gcloud/storage/StorageServiceOptions.java | 21 ++++++-------
 .../DatastoreServiceOptionsTest.java          | 10 ++++++-
 5 files changed, 49 insertions(+), 37 deletions(-)

diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 896d5c1572f5..76419d596958 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -24,6 +24,7 @@
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.gcloud.spi.ServiceRpcFactory;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -34,7 +35,7 @@
 import java.net.URLConnection;
 import java.util.Set;
 
-public abstract class ServiceOptions implements Serializable {
+public abstract class ServiceOptions> implements Serializable {
 
   private static final String DEFAULT_HOST = "https://www.googleapis.com";
   private static final long serialVersionUID = 1203687993961393350L;
@@ -43,6 +44,7 @@ public abstract class ServiceOptions implements Serializable {
   private final HttpTransportFactory httpTransportFactory;
   private final AuthCredentials authCredentials;
   private final RetryParams retryParams;
+  private final ServiceRpcFactory serviceRpcFactory;
 
   public interface HttpTransportFactory extends Serializable {
     HttpTransport create();
@@ -72,12 +74,14 @@ public HttpTransport create() {
     }
   }
 
-  protected abstract static class Builder> {
+  protected abstract static class Builder,
+      B extends Builder> {
 
     private String host;
     private HttpTransportFactory httpTransportFactory;
     private AuthCredentials authCredentials;
     private RetryParams retryParams;
+    private ServiceRpcFactory serviceRpcFactory;
 
     protected Builder() {}
 
@@ -86,6 +90,7 @@ protected Builder(ServiceOptions options) {
       httpTransportFactory = options.httpTransportFactory;
       authCredentials = options.authCredentials;
       retryParams = options.retryParams;
+      serviceRpcFactory = options.serviceRpcFactory;
     }
 
     protected abstract ServiceOptions build();
@@ -114,14 +119,20 @@ public B retryParams(RetryParams retryParams) {
       this.retryParams = retryParams;
       return self();
     }
+
+    public B serviceRpcFactory(ServiceRpcFactory serviceRpcFactory) {
+      this.serviceRpcFactory = serviceRpcFactory;
+      return self();
+    }
   }
 
-  protected ServiceOptions(Builder builder) {
+  protected ServiceOptions(Builder builder) {
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransportFactory =
         firstNonNull(builder.httpTransportFactory, new DefaultHttpTransportFactory());
     authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials());
     retryParams = builder.retryParams;
+    serviceRpcFactory = builder.serviceRpcFactory;
   }
 
   private static AuthCredentials defaultAuthCredentials() {
@@ -204,10 +215,14 @@ public RetryParams retryParams() {
     return retryParams;
   }
 
+  public ServiceRpcFactory serviceRpcFactory() {
+    return serviceRpcFactory;
+  }
+
   public HttpRequestInitializer httpRequestInitializer() {
     HttpTransport httpTransport = httpTransportFactory.create();
     return authCredentials().httpRequestInitializer(httpTransport, scopes());
   }
 
-  public abstract Builder toBuilder();
+  public abstract Builder toBuilder();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index bc7838df4a6b..de369d33f214 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -34,25 +34,25 @@
 import java.util.Iterator;
 import java.util.Set;
 
-public class DatastoreServiceOptions extends ServiceOptions {
+public class DatastoreServiceOptions extends ServiceOptions {
 
+  private static final long serialVersionUID = -8636602944160689193L;
   private static final String DATASET_ENV_NAME = "DATASTORE_DATASET";
   private static final String DATASTORE_SCOPE = "https://www.googleapis.com/auth/datastore";
   private static final String USERINFO_SCOPE = "https://www.googleapis.com/auth/userinfo.email";
   private static final Set SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE);
+
   private final String dataset;
   private final String namespace;
   private final boolean force;
-  private final DatastoreRpc datastoreRpc;
   private final boolean normalizeDataset;
-  private final boolean defaultDatastoreRpc;
 
-  public static class Builder extends ServiceOptions.Builder {
+  public static class Builder extends
+      ServiceOptions.Builder {
 
     private String dataset;
     private String namespace;
     private boolean force;
-    private DatastoreRpc datastoreRpc;
     private boolean normalizeDataset = true;
 
     private Builder() {
@@ -63,7 +63,6 @@ private Builder(DatastoreServiceOptions options) {
       dataset = options.dataset;
       force = options.force;
       namespace = options.namespace;
-      datastoreRpc = options.datastoreRpc;
       normalizeDataset = options.normalizeDataset;
     }
 
@@ -73,11 +72,6 @@ public DatastoreServiceOptions build() {
       return normalizeDataset ? options.normalize() : options;
     }
 
-    public Builder datastoreRpc(DatastoreRpc datastoreRpc) {
-      this.datastoreRpc = datastoreRpc;
-      return this;
-    }
-
     public Builder dataset(String dataset) {
       this.dataset = validateDataset(dataset);
       return this;
@@ -105,8 +99,6 @@ private DatastoreServiceOptions(Builder builder) {
     namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
     force = builder.force;
     dataset = firstNonNull(builder.dataset, defaultDataset());
-    datastoreRpc = firstNonNull(builder.datastoreRpc, ServiceRpcProvider.datastore(this));
-    defaultDatastoreRpc = builder.datastoreRpc == null;
   }
 
   private DatastoreServiceOptions normalize() {
@@ -123,7 +115,7 @@ private DatastoreServiceOptions normalize() {
         .build();
     requestPb.addKey(key);
     try {
-      LookupResponse responsePb = datastoreRpc.lookup(requestPb.build());
+      LookupResponse responsePb = datastoreRpc().lookup(requestPb.build());
       if (responsePb.getDeferredCount() > 0) {
         key = responsePb.getDeferred(0);
       } else {
@@ -132,9 +124,6 @@ private DatastoreServiceOptions normalize() {
         key = combinedIter.next().getEntity().getKey();
       }
       builder.dataset(key.getPartitionId().getDatasetId());
-      if (defaultDatastoreRpc && !dataset.equals(builder.dataset)) {
-        builder.datastoreRpc(ServiceRpcProvider.datastore(builder.build()));
-      }
       return new DatastoreServiceOptions(builder);
     } catch (DatastoreRpcException e) {
       throw DatastoreServiceException.translateAndThrow(e);
@@ -185,8 +174,11 @@ public Builder toBuilder() {
     return new Builder(this);
   }
 
-  public DatastoreRpc datastoreRpc() {
-    return datastoreRpc;
+  DatastoreRpc datastoreRpc() {
+    if (serviceRpcFactory() != null) {
+      return serviceRpcFactory().create(this);
+    }
+    return ServiceRpcProvider.datastore(this);
   }
 
   public static Builder builder() {
diff --git a/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java b/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
index 42c8da0f20f4..a16d11217bb1 100644
--- a/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
+++ b/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

public interface ServiceRpcFactory {

  S create(O options);
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

import java.io.Serializable;

public interface ServiceRpcFactory extends Serializable {

  S create(O options);
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index c36d28c1293f..8894cadad7d3 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -25,22 +25,22 @@
 
 import java.util.Set;
 
-public class StorageServiceOptions extends ServiceOptions {
+public class StorageServiceOptions extends ServiceOptions {
 
+  private static final long serialVersionUID = -7804860602287801084L;
   private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control";
   private static final Set SCOPES = ImmutableSet.of(GCS_SCOPE);
   private static final String DEFAULT_PATH_DELIMITER = "/";
   private static final String PROJECT_ENV_NAME = "default_project_id";
 
-  private final StorageRpc storageRpc;
   private final String project;
   private final String pathDelimiter;
 
-  public static class Builder extends ServiceOptions.Builder {
+  public static class Builder extends
+      ServiceOptions.Builder {
 
     private String project;
     private String pathDelimiter;
-    private StorageRpc storageRpc;
 
     private Builder() {}
 
@@ -58,11 +58,6 @@ public Builder pathDelimiter(String pathDelimiter) {
       return this;
     }
 
-    public Builder storageRpc(StorageRpc storageRpc) {
-      this.storageRpc = storageRpc;
-      return this;
-    }
-
     @Override
     public StorageServiceOptions build() {
       return new StorageServiceOptions(this);
@@ -74,7 +69,6 @@ private StorageServiceOptions(Builder builder) {
     pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
     project = builder.project != null ? builder.project : defaultProject();
     Preconditions.checkArgument(project != null, "Missing required project id");
-    storageRpc = MoreObjects.firstNonNull(builder.storageRpc, ServiceRpcProvider.storage(this));
   }
 
   @Override
@@ -82,8 +76,11 @@ protected Set scopes() {
     return SCOPES;
   }
 
-  public StorageRpc storageRpc() {
-    return storageRpc;
+  StorageRpc storageRpc() {
+    if (serviceRpcFactory() != null) {
+      return serviceRpcFactory().create(this);
+    }
+    return ServiceRpcProvider.storage(this);
   }
 
   public String project() {
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
index c847d3504580..7274f1ca9e91 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -23,6 +23,7 @@
 import static org.junit.Assert.assertTrue;
 
 import com.google.gcloud.spi.DatastoreRpc;
+import com.google.gcloud.spi.DatastoreRpcFactory;
 
 import org.easymock.EasyMock;
 import org.junit.Before;
@@ -33,17 +34,23 @@
 public class DatastoreServiceOptionsTest {
 
   private static final String DATASET = "dataset";
+  private DatastoreRpcFactory datastoreRpcFactory;
   private DatastoreRpc datastoreRpc;
   private DatastoreServiceOptions.Builder options;
 
   @Before
   public void setUp() throws IOException, InterruptedException {
+    datastoreRpcFactory = EasyMock.createMock(DatastoreRpcFactory.class);
     datastoreRpc = EasyMock.createMock(DatastoreRpc.class);
     options = DatastoreServiceOptions.builder()
         .normalizeDataset(false)
-        .datastoreRpc(datastoreRpc)
+        .serviceRpcFactory(datastoreRpcFactory)
         .dataset(DATASET)
         .host("http://localhost:" + LocalGcdHelper.PORT);
+    EasyMock.expect(datastoreRpcFactory.create(EasyMock.anyObject(DatastoreServiceOptions.class)))
+        .andReturn(datastoreRpc)
+        .anyTimes();
+    EasyMock.replay(datastoreRpcFactory, datastoreRpc);
   }
 
   @Test
@@ -70,6 +77,7 @@ public void testForce() throws Exception {
 
   @Test
   public void testDatastore() throws Exception {
+    assertSame(datastoreRpcFactory, options.build().serviceRpcFactory());
     assertSame(datastoreRpc, options.build().datastoreRpc());
   }
 

From b44cdd42befb805ed5992239cce452b65d9ab7c8 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 24 Apr 2015 16:13:45 -0700
Subject: [PATCH 181/732] work in progress

---
 .../com/google/gcloud/AuthCredentials.java    | 29 +++++++-
 .../com/google/gcloud/ServiceOptions.java     | 23 ++++--
 .../datastore/DatastoreServiceOptions.java    | 18 +++++
 .../com/google/gcloud/datastore/Value.java    |  2 +-
 .../gcloud/storage/StorageServiceOptions.java | 16 +++++
 .../gcloud/datastore/SerializationTest.java   | 26 ++++++-
 .../gcloud/storage/SerializationTest.java     | 71 +++++++++++++++++++
 7 files changed, 176 insertions(+), 9 deletions(-)
 create mode 100644 src/test/java/com/google/gcloud/storage/SerializationTest.java

diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java
index e6c783565d8f..839da54e62cf 100644
--- a/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -31,9 +31,11 @@
 
 import java.io.IOException;
 import java.io.ObjectInputStream;
+import java.io.ObjectStreamException;
 import java.io.Serializable;
 import java.security.GeneralSecurityException;
 import java.security.PrivateKey;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -47,11 +49,17 @@ private static class AppEngineAuthCredentials extends AuthCredentials {
 
     private static final long serialVersionUID = 7931300552744202954L;
 
+    private static final AuthCredentials INSTANCE = new AppEngineAuthCredentials();
+
     @Override
     protected HttpRequestInitializer httpRequestInitializer(HttpTransport transport,
         Set scopes) {
       return new AppIdentityCredential(scopes);
     }
+
+    private Object readResolve() throws ObjectStreamException {
+      return INSTANCE;
+    }
   }
 
   private static class ServiceAccountAuthCredentials extends AuthCredentials {
@@ -60,6 +68,8 @@ private static class ServiceAccountAuthCredentials extends AuthCredentials {
     private final String account;
     private final PrivateKey privateKey;
 
+    private static final AuthCredentials NO_CREDENTIALS = new ServiceAccountAuthCredentials();
+
     ServiceAccountAuthCredentials(String account, PrivateKey privateKey) {
       this.account = checkNotNull(account);
       this.privateKey = checkNotNull(privateKey);
@@ -83,6 +93,21 @@ protected HttpRequestInitializer httpRequestInitializer(
       }
       return builder.build();
     }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(account, privateKey);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!(obj instanceof ServiceAccountAuthCredentials)) {
+        return false;
+      }
+      ServiceAccountAuthCredentials other = (ServiceAccountAuthCredentials) obj;
+      return Objects.equals(account, other.account)
+          && Objects.equals(privateKey, other.privateKey);
+    }
   }
 
   private static class ComputeEngineAuthCredentials extends AuthCredentials {
@@ -137,7 +162,7 @@ protected abstract HttpRequestInitializer httpRequestInitializer(HttpTransport t
       Set scopes);
 
   public static AuthCredentials createForAppEngine() {
-    return new AppEngineAuthCredentials();
+    return AppEngineAuthCredentials.INSTANCE;
   }
 
   public static AuthCredentials createForComputeEngine()
@@ -167,7 +192,7 @@ public static AuthCredentials createFor(String account, PrivateKey privateKey) {
   }
 
   public static AuthCredentials noCredentials() {
-    return new ServiceAccountAuthCredentials();
+    return ServiceAccountAuthCredentials.NO_CREDENTIALS;
   }
 
   static ComputeCredential getComputeCredential() throws IOException, GeneralSecurityException {
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 76419d596958..ea3d37f80b9e 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -33,6 +33,7 @@
 import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLConnection;
+import java.util.Objects;
 import java.util.Set;
 
 public abstract class ServiceOptions> implements Serializable {
@@ -50,9 +51,9 @@ public interface HttpTransportFactory extends Serializable {
     HttpTransport create();
   }
 
-  private static class DefaultHttpTransportFactory implements HttpTransportFactory {
+  private enum DefaultHttpTransportFactory implements HttpTransportFactory {
 
-    private static final long serialVersionUID = 2684907202489506911L;
+    INSTANCE;
 
     @Override
     public HttpTransport create() {
@@ -85,7 +86,7 @@ protected abstract static class Builder,
 
     protected Builder() {}
 
-    protected Builder(ServiceOptions options) {
+    protected Builder(ServiceOptions options) {
       host = options.host;
       httpTransportFactory = options.httpTransportFactory;
       authCredentials = options.authCredentials;
@@ -129,7 +130,7 @@ public B serviceRpcFactory(ServiceRpcFactory serviceRpcFactory) {
   protected ServiceOptions(Builder builder) {
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransportFactory =
-        firstNonNull(builder.httpTransportFactory, new DefaultHttpTransportFactory());
+        firstNonNull(builder.httpTransportFactory, DefaultHttpTransportFactory.INSTANCE);
     authCredentials = firstNonNull(builder.authCredentials, defaultAuthCredentials());
     retryParams = builder.retryParams;
     serviceRpcFactory = builder.serviceRpcFactory;
@@ -224,5 +225,19 @@ public HttpRequestInitializer httpRequestInitializer() {
     return authCredentials().httpRequestInitializer(httpTransport, scopes());
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(host, httpTransportFactory, authCredentials, retryParams,
+        serviceRpcFactory);
+  }
+
+  protected boolean isEquals(ServiceOptions other) {
+    return Objects.equals(host, other.host)
+        && Objects.equals(httpTransportFactory, other.httpTransportFactory)
+        && Objects.equals(authCredentials, other.authCredentials)
+        && Objects.equals(retryParams, other.retryParams)
+        && Objects.equals(serviceRpcFactory, other.serviceRpcFactory);
+  }
+
   public abstract Builder toBuilder();
 }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index de369d33f214..8d5f062fd548 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -32,6 +32,7 @@
 
 import java.lang.reflect.Method;
 import java.util.Iterator;
+import java.util.Objects;
 import java.util.Set;
 
 public class DatastoreServiceOptions extends ServiceOptions {
@@ -174,6 +175,23 @@ public Builder toBuilder() {
     return new Builder(this);
   }
 
+  @Override
+  public int hashCode() {
+    return super.hashCode() ^ Objects.hash(dataset, namespace, force, normalizeDataset);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof DatastoreServiceOptions)) {
+      return false;
+    }
+    DatastoreServiceOptions other = (DatastoreServiceOptions) obj;
+    return isEquals(other) && Objects.equals(dataset, other.dataset)
+        && Objects.equals(namespace, other.namespace)
+        && Objects.equals(force, other.force)
+        && Objects.equals(normalizeDataset, other.normalizeDataset);
+  }
+
   DatastoreRpc datastoreRpc() {
     if (serviceRpcFactory() != null) {
       return serviceRpcFactory().create(this);
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/src/main/java/com/google/gcloud/datastore/Value.java
index 60d1468cb90c..c5fc63a960b1 100644
--- a/src/main/java/com/google/gcloud/datastore/Value.java
+++ b/src/main/java/com/google/gcloud/datastore/Value.java
@@ -85,6 +85,7 @@ public final DatastoreV1.Value toProto(P value) {
     protected abstract void setValue(P from, DatastoreV1.Value.Builder to);
   }
 
+  @SuppressWarnings("deprecation")
   abstract static class BaseBuilder, B extends BaseBuilder>
       implements ValueBuilder {
 
@@ -102,7 +103,6 @@ public ValueType getValueType() {
       return valueType;
     }
 
-    @SuppressWarnings("deprecation")
     @Override
     public B mergeFrom(P other) {
       indexed = other.indexed();
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 8894cadad7d3..a08edeee7819 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -23,6 +23,7 @@
 import com.google.gcloud.spi.ServiceRpcProvider;
 import com.google.gcloud.spi.StorageRpc;
 
+import java.util.Objects;
 import java.util.Set;
 
 public class StorageServiceOptions extends ServiceOptions {
@@ -96,6 +97,21 @@ public Builder toBuilder() {
     return new Builder(this);
   }
 
+  @Override
+  public int hashCode() {
+    return super.hashCode() ^ Objects.hash(project, pathDelimiter);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof StorageServiceOptions)) {
+      return false;
+    }
+    StorageServiceOptions other = (StorageServiceOptions) obj;
+    return isEquals(other) && Objects.equals(project, other.project)
+        && Objects.equals(pathDelimiter, other.pathDelimiter);
+  }
+
   private static String defaultProject() {
     String projectId = System.getProperty(PROJECT_ENV_NAME, System.getenv(PROJECT_ENV_NAME));
     if (projectId == null) {
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 870aee8add27..ed0da1e9ef13 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -23,6 +23,8 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Multimap;
+import com.google.gcloud.AuthCredentials;
+import com.google.gcloud.RetryParams;
 import com.google.gcloud.datastore.StructuredQuery.CompositeFilter;
 import com.google.gcloud.datastore.StructuredQuery.OrderBy;
 import com.google.gcloud.datastore.StructuredQuery.Projection;
@@ -47,7 +49,7 @@ public class SerializationTest {
   private static final DateTime DATE_TIME1 = DateTime.now();
   private static final Blob BLOB1 = Blob.copyFrom(UTF_8.encode("hello world"));
   private static final Cursor CURSOR1 = Cursor.copyFrom(new byte[] {1,2});
-  private static final Cursor CURSOR2 = Cursor.copyFrom(new byte[] {10});
+  private static final Cursor CURSOR2 = Cursor.copyFrom(new byte[]{10});
   private static final Query GQL1 =
       Query.gqlQueryBuilder("select * from kind1 where name = @name and age > @1")
       .setBinding("name", "name1")
@@ -128,6 +130,26 @@ public class SerializationTest {
       .put(ValueType.RAW_VALUE, RAW_VALUE)
       .build();
 
+  @Test
+  public void testServiceOptions() throws Exception {
+    DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+        .authCredentials(AuthCredentials.createForAppEngine())
+        .normalizeDataset(false)
+        .dataset("ds1")
+        .build();
+    DatastoreServiceOptions serializedCopy = serializeAndDeserialize(options);
+    assertEquals(options, serializedCopy);
+
+    options = options.toBuilder()
+        .namespace("ns1")
+        .retryParams(RetryParams.getDefaultInstance())
+        .authCredentials(AuthCredentials.noCredentials())
+        .force(true)
+        .build();
+    serializedCopy = serializeAndDeserialize(options);
+    assertEquals(options, serializedCopy);
+  }
+
   @Test
   public void testValues() throws Exception {
     for (ValueType valueType : ValueType.values()) {
@@ -157,7 +179,7 @@ public void testTypes() throws Exception {
   }
 
   @SuppressWarnings("unchecked")
-  private  T serializeAndDeserialize(T obj)
+  private  T serializeAndDeserialize(T obj)
       throws IOException, ClassNotFoundException {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     try (ObjectOutputStream output = new ObjectOutputStream(bytes)) {
diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/src/test/java/com/google/gcloud/storage/SerializationTest.java
new file mode 100644
index 000000000000..dc340c01480a
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.gcloud.AuthCredentials;
+import com.google.gcloud.RetryParams;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+public class SerializationTest {
+
+
+  @Test
+  public void testServiceOptions() throws Exception {
+    StorageServiceOptions options = StorageServiceOptions.builder()
+        .project("p1")
+        .authCredentials(AuthCredentials.createForAppEngine())
+        .build();
+    StorageServiceOptions serializedCopy = serializeAndDeserialize(options);
+    assertEquals(options, serializedCopy);
+
+    options = options.toBuilder()
+        .project("p2")
+        .retryParams(RetryParams.getDefaultInstance())
+        .authCredentials(AuthCredentials.noCredentials())
+        .pathDelimiter(":")
+        .build();
+    serializedCopy = serializeAndDeserialize(options);
+    assertEquals(options, serializedCopy);
+  }
+
+  @Test
+  public void testTypes() throws Exception {
+    // todo: implement
+  }
+
+  @SuppressWarnings("unchecked")
+  private  T serializeAndDeserialize(T obj)
+      throws IOException, ClassNotFoundException {
+    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+    try (ObjectOutputStream output = new ObjectOutputStream(bytes)) {
+      output.writeObject(obj);
+    }
+    try (ObjectInputStream input =
+        new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
+      return (T) input.readObject();
+    }
+  }
+}

From d62c008f5118fb33e9b8daa0d4be53c9fc1eb5d6 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 24 Apr 2015 17:44:06 -0700
Subject: [PATCH 182/732] work in progress

---
 .../gcloud/examples/StorageExample.java       |  2 +-
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../gcloud/storage/BlobIterOptions.java       |  1 +
 .../com/google/gcloud/storage/BlobList.java   | 42 +++++++++++++++++++
 .../gcloud/storage/BlobListOptions.java       | 37 ++++++++++++++++
 .../google/gcloud/storage/ListOptions.java    |  1 -
 .../google/gcloud/storage/StorageService.java |  7 +++-
 .../gcloud/storage/StorageServiceImpl.java    |  2 +-
 9 files changed, 89 insertions(+), 7 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/BlobIterOptions.java
 create mode 100644 src/main/java/com/google/gcloud/storage/BlobList.java
 create mode 100644 src/main/java/com/google/gcloud/storage/BlobListOptions.java
 delete mode 100644 src/main/java/com/google/gcloud/storage/ListOptions.java

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index f65c8cdd02c2..071c7fad2de7 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -149,7 +149,7 @@ public void run(StorageService storage, String bucket) {
           System.out.println(buckets.next());
         }
       } else {
-        Iterator blobs = storage.list(bucket, ListOptions.of());
+        Iterator blobs = storage.list(bucket, BlobIterOptions.of());
         while (blobs.hasNext()) {
           System.out.println(blobs.next());
         }
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 5451375e64f8..e3fd45e528e0 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    try {
      return storage.buckets()
          .list(options.project())
          .setProjection(DEFAULT_PROJECTION)
          .execute()
          .getItems()
          .iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, int limit) {
    // todo: implement
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    // todo: missing setProjection
    try {
      return storage.objects()
          .compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {

    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    // todo: paging
    try {
      return storage.buckets()
          .list(options.project())
          .setProjection(DEFAULT_PROJECTION)
          .execute()
          .getItems()
          .iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, long limit) {
    // todo: implement
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(includeOlderVersions)
          .setDelimiter(delimiter)
          .setPrefix(prefix)
          .setMaxResults(limit)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    // todo: missing setProjection
    try {
      return storage.objects()
          .compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {

    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 17646392686f..af90731ac08d 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationMatch");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Iterator list() throws StorageServiceException;

  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, int limit) throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  void delete(Bucket bucket, Map options) throws StorageServiceException;

  void delete(StorageObject object, Map options) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationMatch");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Iterator list() throws StorageServiceException;

  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, long limit) throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  void delete(Bucket bucket, Map options) throws StorageServiceException;

  void delete(StorageObject object, Map options) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BlobIterOptions.java b/src/main/java/com/google/gcloud/storage/BlobIterOptions.java
new file mode 100644
index 000000000000..9ad1eeda85cb
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BlobIterOptions.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class BlobIterOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    Builder() {}

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }

    public BlobIterOptions build() {
      return new BlobIterOptions(this);
    }
  }

  BlobIterOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }

  public static BlobIterOptions of() {
    return builder().build();
  }

  public static Builder builder() {
    return new Builder();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BlobList.java b/src/main/java/com/google/gcloud/storage/BlobList.java
new file mode 100644
index 000000000000..c11c671f96e8
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BlobList.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Created by ozarov on 4/24/15.
+ */
+public class BlobList  implements Serializable{
+
+  private final List blobs;
+  private final String cursor;
+
+  BlobList(List blobs, String cursor) {
+    this.blobs = blobs;
+    this.cursor = cursor;
+  }
+
+  public List result() {
+    return blobs;
+  }
+
+  public String cursor() {
+    return cursor;
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/BlobListOptions.java b/src/main/java/com/google/gcloud/storage/BlobListOptions.java
new file mode 100644
index 000000000000..236f68f485cf
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BlobListOptions.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+/**
+ * Created by ozarov on 4/24/15.
+ */
+public class BlobListOptions extends BlobIterOptions {
+
+  private final String cursor;
+
+  public Builder extends BlobIterOptions.Builder {
+
+  }
+
+  private BlobListOptions(Builder builder) {
+    super(builder);
+  }
+
+  public String cursor() {
+    return cursor;
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/ListOptions.java b/src/main/java/com/google/gcloud/storage/ListOptions.java
deleted file mode 100644
index d571b176d9ab..000000000000
--- a/src/main/java/com/google/gcloud/storage/ListOptions.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class ListOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final String cursor;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    private Builder() {}

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }

    public ListOptions build() {
      return new ListOptions(this);
    }
  }

  private ListOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    cursor = builder.cursor;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public String cursor() {
    return cursor;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }

  public static ListOptions of() {
    return builder().build();
  }

  public static Builder builder() {
    return new Builder();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 2fb92cd66743..6d81dc5b8539 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -325,12 +325,15 @@ public static Builder builder() {
   /**
    * @throws StorageServiceException upon failure
    */
-  Iterator list();
+  Iterator buckets();
 
   /**
    * @throws StorageServiceException upon failure
    */
-  Iterator list(String bucket, ListOptions settings);
+  Iterator blobs(String bucket, BlobIterOptions settings);
+
+
+  BlobList blobs(String bucket, BlobListOptions settings);
 
   /**
    * @throws StorageServiceException upon failure
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index afaca940c9f0..2d6bc68dafa9 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -134,7 +134,7 @@ public Iterator call() {
   }
 
   @Override
-  public Iterator list(String bucket, ListOptions settings) {
+  public Iterator list(String bucket, BlobIterOptions settings) {
     // todo implement paging (with retries) with if limit is not given or > X
     String delimiter = settings.recursive() ? options().pathDelimiter() : null;
     return Iterators.transform(

From 5d7eaf0d968f19c1c56abfe8360ba19b5fff10e7 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 28 Apr 2015 17:37:36 -0700
Subject: [PATCH 183/732] work in progress

---
 pom.xml                                       |   2 +-
 .../gcloud/examples/StorageExample.java       | 118 +++++++++-----
 .../google/gcloud/spi/DefaultStorageRpc.java  |   2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../java/com/google/gcloud/storage/Acl.java   |   5 +
 .../gcloud/storage/BlobIterOptions.java       |   1 -
 .../gcloud/storage/BlobListOptions.java       |  37 -----
 .../com/google/gcloud/storage/Bucket.java     |  19 ++-
 .../{BlobList.java => ListResult.java}        |  23 +--
 .../google/gcloud/storage/StorageService.java | 123 +++++++++-----
 .../gcloud/storage/StorageServiceImpl.java    | 152 ++++++++++++------
 .../com/google/gcloud/storage/Validator.java  |  54 -------
 12 files changed, 302 insertions(+), 236 deletions(-)
 delete mode 100644 src/main/java/com/google/gcloud/storage/BlobIterOptions.java
 delete mode 100644 src/main/java/com/google/gcloud/storage/BlobListOptions.java
 rename src/main/java/com/google/gcloud/storage/{BlobList.java => ListResult.java} (63%)
 delete mode 100644 src/main/java/com/google/gcloud/storage/Validator.java

diff --git a/pom.xml b/pom.xml
index 06b070ecf874..c09da7963c42 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,7 +115,7 @@
     
       com.google.apis
       google-api-services-storage
-      v1-rev23-1.19.0
+      v1-rev33-1.20.0
       compile
     
     
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 071c7fad2de7..b06a99217b99 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,7 +16,14 @@
 
 package com.google.gcloud.examples;
 
-import com.google.gcloud.storage.*;
+import com.google.gcloud.spi.StorageRpc.Tuple;
+import com.google.gcloud.storage.Blob;
+import com.google.gcloud.storage.Bucket;
+import com.google.gcloud.storage.StorageService;
+import com.google.gcloud.storage.StorageService.CopyRequest;
+import com.google.gcloud.storage.StorageService.ComposeRequest;
+import com.google.gcloud.storage.StorageServiceFactory;
+import com.google.gcloud.storage.StorageServiceOptions;
 
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -24,7 +31,6 @@
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.Map;
 
 /**
@@ -38,7 +44,9 @@
  * 
    1. compile using maven - {@code mvn compile}
      2. 
  * 
    2. run using maven -
  * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
- * -Dexec.args="project_id list []|info [ []]|get  |upload   []|delete  "}
+ * -Dexec.args="project_id list []| info [ []]| get  |
+ *  upload   []| delete  |
+ *  cp    | compose  + "}
  * 
      3. 
  * 
    
  */
@@ -57,30 +65,6 @@ protected String params() {
     }
   }
 
-  private static class Tuple {
-
-    private final X x;
-    private final Y y;
-
-    private Tuple(X x, Y y) {
-      this.x = x;
-      this.y = y;
-    }
-
-    static  Tuple of(X x, Y y) {
-      return new Tuple<>(x, y);
-    }
-
-    X first() {
-      return x;
-    }
-
-    Y second() {
-      return y;
-    }
-  }
-
-
   private static abstract class BlobAction extends StorageAction {
 
     @Override
@@ -144,14 +128,12 @@ String parse(String... args) {
     @Override
     public void run(StorageService storage, String bucket) {
       if (bucket == null) {
-        Iterator buckets = storage.list();
-        while (buckets.hasNext()) {
-          System.out.println(buckets.next());
+        for (Bucket b : storage.list()) {
+          System.out.println(b);
         }
       } else {
-        Iterator blobs = storage.list(bucket, BlobIterOptions.of());
-        while (blobs.hasNext()) {
-          System.out.println(blobs.next());
+        for (Blob b : storage.list(bucket)) {
+          System.out.println(b);
         }
       }
     }
@@ -165,12 +147,12 @@ public String params() {
   private static class UploadAction extends StorageAction> {
     @Override
     public void run(StorageService storage, Tuple tuple) throws Exception {
-      if (Files.size(tuple.first()) > 1_000_000) {
+      if (Files.size(tuple.x()) > 1_000_000) {
         // todo: upload via streaming API
         throw new IllegalArgumentException("file is too big");
       } else {
-        byte[] bytes = Files.readAllBytes(tuple.first());
-        System.out.println(storage.create(tuple.second(), bytes));
+        byte[] bytes = Files.readAllBytes(tuple.x());
+        System.out.println(storage.create(tuple.y(), bytes));
       }
     }
 
@@ -196,6 +178,7 @@ public void run(StorageService storage, Blob blob) {
       blob = storage.get(blob);
       if (blob == null) {
         System.out.println("No such object");
+        return;
       }
       if (blob.size() < 1_000_000) {
         System.out.println(new String(storage.load(blob), StandardCharsets.UTF_8));
@@ -206,27 +189,73 @@ public void run(StorageService storage, Blob blob) {
     }
   }
 
+  private static class CopyAction extends StorageAction {
+    @Override
+    public void run(StorageService storage, CopyRequest request) {
+      System.out.println(storage.copy(request));
+    }
+
+    @Override
+    CopyRequest parse(String... args) {
+      if (args.length != 4) {
+        throw new IllegalArgumentException();
+      }
+      return CopyRequest.of(Blob.of(args[0], args[1]), Blob.of(args[2], args[3]));
+    }
+
+    @Override
+    public String params() {
+      return "   ";
+    }
+  }
+
+  private static class ComposeAction extends StorageAction {
+    @Override
+    public void run(StorageService storage, ComposeRequest request) {
+      System.out.println(storage.compose(request));
+    }
+
+    @Override
+    ComposeRequest parse(String... args) {
+      if (args.length < 3) {
+        throw new IllegalArgumentException();
+      }
+      ComposeRequest.Builder request = ComposeRequest.builder();
+      request.target(Blob.of(args[0], args[args.length - 1]));
+      for (int i = 1; i < args.length - 1; i++) {
+        request.addSource(args[i]);
+      }
+      return request.build();
+    }
+
+    @Override
+    public String params() {
+      return " + ";
+    }
+  }
+
   static {
     ACTIONS.put("info", new InfoAction());
     ACTIONS.put("delete", new DeleteAction());
     ACTIONS.put("list", new ListAction());
     ACTIONS.put("upload", new UploadAction());
     ACTIONS.put("get", new GetAction());
+    ACTIONS.put("cp", new CopyAction());
+    ACTIONS.put("compose", new ComposeAction());
   }
 
   public static void printUsage() {
-    StringBuilder actionAndParams = new StringBuilder();
+    StringBuilder actionAndParams = new StringBuilder("");
     for (Map.Entry entry : ACTIONS.entrySet()) {
-      actionAndParams.append(entry.getKey());
+      actionAndParams.append("\n\t").append(entry.getKey());
 
       String param = entry.getValue().params();
       if (param != null && !param.isEmpty()) {
         actionAndParams.append(' ').append(param);
       }
-      actionAndParams.append('|');
     }
-    actionAndParams.setLength(actionAndParams.length() - 1);
-    System.out.printf("Usage: %s [%s]%n", StorageExample.class.getSimpleName(), actionAndParams);
+    System.out.printf("Usage: %s   *%s%n",
+        StorageExample.class.getSimpleName(), actionAndParams);
   }
 
   @SuppressWarnings("unchecked")
@@ -236,7 +265,6 @@ public static void main(String... args) throws Exception {
       printUsage();
       return;
     }
-    String projectId = args[0];
     StorageAction action = ACTIONS.get(args[1]);
     if (action == null) {
       System.out.println("Unrecognized action '" + args[1] + "'");
@@ -249,10 +277,14 @@ public static void main(String... args) throws Exception {
     Object request;
     try {
       request = action.parse(args);
-    } catch (Exception ex) {
+    } catch (IllegalArgumentException ex) {
       System.out.println("Invalid input for action '" + args[1] + "'");
       System.out.println("Expected: " + action.params());
       return;
+    } catch (Exception ex) {
+      System.out.println("Failed to parse request.");
+      ex.printStackTrace();
+      return;
     }
     action.run(storage, request);
   }
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index e3fd45e528e0..1d999d23d5ef 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  private static final Set RETRYABLE_CODES = ImmutableSet.of(500, 503);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list() {
    // todo: paging
    try {
      return storage.buckets()
          .list(options.project())
          .setProjection(DEFAULT_PROJECTION)
          .execute()
          .getItems()
          .iterator();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Iterator list(String bucket, String prefix, String delimiter,
      String cursor, boolean includeOlderVersions, long limit) {
    // todo: implement
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(includeOlderVersions)
          .setDelimiter(delimiter)
          .setPrefix(prefix)
          .setMaxResults(limit)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return null;
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets().get(bucket.getName()).setProjection(DEFAULT_PROJECTION).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket).setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets().delete(bucket.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects().delete(blob.getBucket(), blob.getName()).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    // todo: implement null -> ComposeRequest
    // todo: missing setProjection
    try {
      return storage.objects()
          .compose(target.getBucket(), target.getName(), null)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), target)
          .setProjection(DEFAULT_PROJECTION)
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects().get(from.getBucket(), from.getName());
      if (from.getGeneration() != null && options.containsKey(Option.IF_GENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_GENERATION_MATCH)) {
          getRequest.setIfGenerationMatch(from.getGeneration());
        } else {
          getRequest.setIfGenerationNotMatch(from.getGeneration());
        }
      }
      if (from.getMetageneration() != null && options.containsKey(Option.IF_METAGENERATION_MATCH)) {
        if ((Boolean) options.get(Option.IF_METAGENERATION_MATCH)) {
          getRequest.setIfMetagenerationMatch(from.getMetageneration());
        } else {
          getRequest.setIfMetagenerationNotMatch(from.getMetageneration());
        }
      }
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {

    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects()
          .delete(blob.getBucket(), blob.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      request.setDestination(target);
    }
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index af90731ac08d..1140fc7469f8 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Iterator;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationMatch");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Iterator list() throws StorageServiceException;

  Iterator list(String bucket, String prefix, String delimiter, String cursor,
      boolean versions, long limit) throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  void delete(Bucket bucket, Map options) throws StorageServiceException;

  void delete(StorageObject object, Map options) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  void delete(Bucket bucket, Map options) throws StorageServiceException;

  void delete(StorageObject object, Map options) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 3058386d7201..10673b5aeae0 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -35,6 +35,7 @@ public enum Role {
   public static abstract class Entity implements Serializable {
 
     private static final long serialVersionUID = -2707407252771255840L;
+
     private final Type type;
     private final String value;
 
@@ -152,6 +153,8 @@ public static User ofAllAuthenticatedUsers() {
 
   public static class Project extends Entity {
 
+    private static final long serialVersionUID = 7933776866530023027L;
+
     private final ProjectRole pRole;
     private final String projectId;
 
@@ -176,6 +179,8 @@ public String projectId() {
 
   public static class RawEntity extends Entity {
 
+    private static final long serialVersionUID = 3966205614223053950L;
+
     RawEntity(String entity) {
       super(Type.UNKNOWN, entity);
     }
diff --git a/src/main/java/com/google/gcloud/storage/BlobIterOptions.java b/src/main/java/com/google/gcloud/storage/BlobIterOptions.java
deleted file mode 100644
index 9ad1eeda85cb..000000000000
--- a/src/main/java/com/google/gcloud/storage/BlobIterOptions.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

import java.io.Serializable;

public class BlobIterOptions implements Serializable {

  private static final long serialVersionUID = -7260643619742432140L;

  private final boolean recursive;
  private final String prefix;
  private final Integer maxResults;
  private final boolean includeOlderVersions;

  public static class Builder {

    private boolean recursive;
    private String prefix;
    private String cursor;
    private int maxResults;
    private boolean includeOlderVersions;

    Builder() {}

    public Builder recursive(boolean recursive) {
      this.recursive = recursive;
      return this;
    }

    public Builder prefix(String prefix) {
      this.prefix = prefix;
      return this;
    }

    public Builder cursor(String cursor) {
      this.cursor = cursor;
      return this;
    }

    public Builder maxResults(Integer maxResults) {
      this.maxResults = maxResults;
      return this;
    }

    public Builder includeOlderVersions(boolean include) {
      this.includeOlderVersions = include;
      return this;
    }

    public BlobIterOptions build() {
      return new BlobIterOptions(this);
    }
  }

  BlobIterOptions(Builder builder) {
    recursive = builder.recursive;
    prefix = builder.prefix;
    maxResults = builder.maxResults;
    includeOlderVersions = builder.includeOlderVersions;
  }

  public boolean recursive() {
    return recursive;
  }

  public String prefix() {
    return prefix;
  }

  public Integer maxResults() {
    return maxResults;
  }

  public boolean includeOlderVersions() {
    return includeOlderVersions;
  }

  public static BlobIterOptions of() {
    return builder().build();
  }

  public static Builder builder() {
    return new Builder();
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BlobListOptions.java b/src/main/java/com/google/gcloud/storage/BlobListOptions.java
deleted file mode 100644
index 236f68f485cf..000000000000
--- a/src/main/java/com/google/gcloud/storage/BlobListOptions.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.storage;
-
-/**
- * Created by ozarov on 4/24/15.
- */
-public class BlobListOptions extends BlobIterOptions {
-
-  private final String cursor;
-
-  public Builder extends BlobIterOptions.Builder {
-
-  }
-
-  private BlobListOptions(Builder builder) {
-    super(builder);
-  }
-
-  public String cursor() {
-    return cursor;
-  }
-}
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 16937fda2b4b..981dc7f788c0 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -19,6 +19,7 @@
 import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.collect.Lists.transform;
 
+import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.Bucket.Lifecycle;
 import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
@@ -33,6 +34,9 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.storage.Acl.Entity;
 
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.util.List;
 
@@ -148,7 +152,9 @@ void populateCondition(Rule.Condition condition) {
 
   static class RawDeleteRule extends DeleteRule {
 
-    private final Rule rule;
+    private static final long serialVersionUID = -7166938278642301933L;
+
+    private transient Rule rule;
 
     RawDeleteRule(Rule rule) {
       super(Type.UNKNOWN);
@@ -159,6 +165,17 @@ void populateCondition(Rule.Condition condition) {
       throw new UnsupportedOperationException();
     }
 
+    private void writeObject(ObjectOutputStream out) throws IOException {
+      out.defaultWriteObject();
+      out.writeUTF(rule.toString());
+    }
+
+    private void readObject(ObjectInputStream in) throws IOException,
+        ClassNotFoundException {
+      in.defaultReadObject();
+      rule = new JacksonFactory().fromString(in.readUTF(), Rule.class);
+    }
+
     Rule toPb() {
       return rule;
     }
diff --git a/src/main/java/com/google/gcloud/storage/BlobList.java b/src/main/java/com/google/gcloud/storage/ListResult.java
similarity index 63%
rename from src/main/java/com/google/gcloud/storage/BlobList.java
rename to src/main/java/com/google/gcloud/storage/ListResult.java
index c11c671f96e8..180e5f7a4155 100644
--- a/src/main/java/com/google/gcloud/storage/BlobList.java
+++ b/src/main/java/com/google/gcloud/storage/ListResult.java
@@ -17,26 +17,29 @@
 package com.google.gcloud.storage;
 
 import java.io.Serializable;
-import java.util.List;
+import java.util.Iterator;
 
 /**
- * Created by ozarov on 4/24/15.
+ * Google Cloud storage list result.
  */
-public class BlobList  implements Serializable{
+public class ListResult implements Iterable, Serializable {
+
+  private static final long serialVersionUID = -6937287874908527950L;
 
-  private final List blobs;
   private final String cursor;
+  private final Iterable results;
 
-  BlobList(List blobs, String cursor) {
-    this.blobs = blobs;
+  ListResult(String cursor, Iterable results) {
     this.cursor = cursor;
+    this.results = results;
   }
 
-  public List result() {
-    return blobs;
+  public String nextPageCursor() {
+    return cursor;
   }
 
-  public String cursor() {
-    return cursor;
+  @Override
+  public Iterator iterator() {
+    return results.iterator();
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 6d81dc5b8539..714343a96c5d 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -17,7 +17,6 @@
 package com.google.gcloud.storage;
 
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.gcloud.storage.Validator.checkBlobOptions;
 
 import com.google.common.collect.ImmutableList;
 import com.google.gcloud.Service;
@@ -26,7 +25,6 @@
 import java.io.Serializable;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -130,43 +128,90 @@ public static BlobSourceOption metagenerationMatch(boolean match) {
     }
   }
 
+  class BucketListOption extends Option {
+
+    private static final long serialVersionUID = 8754017079673290353L;
+
+    private BucketListOption(StorageRpc.Option option, Object value) {
+      super(option, value);
+    }
+
+    public static BucketListOption maxResults(long maxResults) {
+      return new BucketListOption(StorageRpc.Option.MAX_RESULTS, maxResults);
+    }
+
+    public static BucketListOption startPageToken(String pageToken) {
+      return new BucketListOption(StorageRpc.Option.PAGE_TOKEN, pageToken);
+    }
+
+    public static BucketListOption prefix(String prefix) {
+      return new BucketListOption(StorageRpc.Option.PREFIX, prefix);
+    }
+  }
+
+  class BlobListOption extends Option {
+
+    private static final long serialVersionUID = 9083383524788661294L;
+
+    private BlobListOption(StorageRpc.Option option, Object value) {
+      super(option, value);
+    }
+
+    public static BlobListOption maxResults(long maxResults) {
+      return new BlobListOption(StorageRpc.Option.MAX_RESULTS, maxResults);
+    }
+
+    public static BlobListOption startPageToken(String pageToken) {
+      return new BlobListOption(StorageRpc.Option.PAGE_TOKEN, pageToken);
+    }
+
+    public static BlobListOption prefix(String prefix) {
+      return new BlobListOption(StorageRpc.Option.PREFIX, prefix);
+    }
+
+    public static BlobListOption recursive(boolean recursive) {
+      return new BlobListOption(StorageRpc.Option.DELIMITER, recursive);
+    }
+  }
+
   class ComposeRequest implements Serializable {
 
     private static final long serialVersionUID = -7385681353748590911L;
 
-    private final String sourceBucket;
     private final List sourceBlobs;
     private final Blob target;
     private final List targetOptions;
 
-    static class SourceBlob implements Serializable {
+    public static class SourceBlob implements Serializable {
 
       private static final long serialVersionUID = 4094962795951990439L;
 
-      final String blob;
+      final String name;
       final Long generation;
 
-      SourceBlob(String blob) {
-        this(blob, null);
+      SourceBlob(String name) {
+        this(name, null);
       }
 
-      SourceBlob(String blob, Long generation) {
-        this.blob = blob;
+      SourceBlob(String name, Long generation) {
+        this.name = name;
         this.generation = generation;
       }
+
+      public String name() {
+        return name;
+      }
+
+      public Long generation() {
+        return generation;
+      }
     }
 
     public static class Builder {
 
-      private String bucket;
-      private List sourceBlobs = new LinkedList<>();
+      private final List sourceBlobs = new LinkedList<>();
       private Blob target;
-      private Set targetOptions = new LinkedHashSet<>();
-
-      public Builder sourceBucket(String bucket) {
-        this.bucket = bucket;
-        return this;
-      }
+      private final Set targetOptions = new LinkedHashSet<>();
 
       public Builder addSource(Iterable blobs) {
         for (String blob : blobs) {
@@ -195,36 +240,33 @@ public Builder targetOptions(BlobTargetOption... options) {
       }
 
       public ComposeRequest build() {
-        checkNotNull(bucket);
         checkNotNull(target);
-        checkBlobOptions("Target", target, targetOptions);
         return new ComposeRequest(this);
       }
     }
 
     private ComposeRequest(Builder builder) {
-      sourceBucket = builder.bucket;
       sourceBlobs = ImmutableList.copyOf(builder.sourceBlobs);
       target = builder.target;
       targetOptions = ImmutableList.copyOf(builder.targetOptions);
     }
 
-    String sourceBucket() {
-      return sourceBucket;
-    }
-
-    List sourceBlobs() {
+    public List sourceBlobs() {
       return sourceBlobs;
     }
 
-    Blob target() {
+    public Blob target() {
       return target;
     }
 
-    List targetOptions() {
+    public List targetOptions() {
       return targetOptions;
     }
 
+    public static ComposeRequest of(Iterable sources, Blob target) {
+      return builder().target(target).addSource(sources).build();
+    }
+
     public static Builder builder() {
       return new Builder();
     }
@@ -242,12 +284,13 @@ class CopyRequest implements Serializable {
     public static class Builder {
 
       private Blob source;
-      private Set sourceOptions;
+      private final Set sourceOptions = new LinkedHashSet<>();
       private Blob target;
-      private Set targetOptions;
+      private final Set targetOptions = new LinkedHashSet<>();
 
-      public void source(Blob source) {
+      public Builder source(Blob source) {
         this.source = source;
+        return this;
       }
 
       public Builder sourceOptions(BlobSourceOption... options) {
@@ -268,8 +311,6 @@ public Builder targetOptions(BlobTargetOption... options) {
       public CopyRequest build() {
         checkNotNull(source);
         checkNotNull(target);
-        checkBlobOptions("Source", source, sourceOptions);
-        checkBlobOptions("Target", target, targetOptions);
         return new CopyRequest(this);
       }
     }
@@ -281,7 +322,7 @@ private CopyRequest(Builder builder) {
       targetOptions = ImmutableList.copyOf(builder.targetOptions);
     }
 
-    Blob source() {
+    public Blob source() {
       return source;
     }
 
@@ -289,14 +330,18 @@ public List sourceOptions() {
       return sourceOptions;
     }
 
-    Blob target() {
+    public Blob target() {
       return target;
     }
 
-    List targetOptions() {
+    public List targetOptions() {
       return targetOptions;
     }
 
+    public static CopyRequest of(Blob source, Blob target) {
+      return builder().source(source).target(target).build();
+    }
+
     public static Builder builder() {
       return new Builder();
     }
@@ -325,15 +370,13 @@ public static Builder builder() {
   /**
    * @throws StorageServiceException upon failure
    */
-  Iterator buckets();
+  ListResult list(BucketListOption... options);
 
   /**
+   * Lists blobs for a bucket.
    * @throws StorageServiceException upon failure
    */
-  Iterator blobs(String bucket, BlobIterOptions settings);
-
-
-  BlobList blobs(String bucket, BlobListOptions settings);
+  ListResult list(String bucket, BlobListOption... options);
 
   /**
    * @throws StorageServiceException upon failure
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 2d6bc68dafa9..5288c064e626 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,21 +16,24 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.gcloud.RetryHelper.runWithRetries;
 
 import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterators;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
 import com.google.gcloud.ExceptionHandler.Interceptor;
 import com.google.gcloud.RetryParams;
 import com.google.gcloud.spi.StorageRpc;
+import com.google.gcloud.spi.StorageRpc.Tuple;
 
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
@@ -70,9 +73,8 @@ public RetryResult beforeEval(Exception exception) {
 
   @Override
   public Bucket create(Bucket bucket, BucketTargetOption... options) {
-    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(bucket, options);
     return Bucket.fromPb(runWithRetries(
         new Callable() {
           @Override
@@ -84,9 +86,8 @@ public com.google.api.services.storage.model.Bucket call() {
 
   @Override
   public Blob create(Blob blob, final byte[] content, BlobTargetOption... options) {
-    Validator.checkBlobOptions(blob, options);
     final StorageObject blobPb = blob.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(blob, options);
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
@@ -97,9 +98,8 @@ public StorageObject call() {
 
   @Override
   public Bucket get(Bucket bucket, BucketSourceOption... options) {
-    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(bucket, options);
     return Bucket.fromPb(runWithRetries(
         new Callable() {
           @Override
@@ -111,9 +111,8 @@ public com.google.api.services.storage.model.Bucket call() {
 
   @Override
   public Blob get(Blob blob, BlobSourceOption... options) {
-    Validator.checkBlobOptions(blob, options);
     final StorageObject storedObject = blob.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(blob, options);
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
@@ -123,30 +122,47 @@ public StorageObject call() {
   }
 
   @Override
-  public Iterator list() {
-    return Iterators.transform(
-        runWithRetries(new Callable>() {
+  public ListResult list(BucketListOption... options) {
+    final Map optionsMap = optionMap(null, null, options);
+    Tuple> result = runWithRetries(
+        new Callable>>() {
           @Override
-          public Iterator call() {
-            return storageRpc.list();
+          public Tuple> call() {
+            return storageRpc.list(optionsMap);
           }
-        }, retryParams, EXCEPTION_HANDLER), Bucket.FROM_PB_FUNCTION);
+        }, retryParams, EXCEPTION_HANDLER);
+    return new ListResult<>(result.x(), Iterables.transform(result.y(),
+        new Function() {
+          @Override
+          public Bucket apply(com.google.api.services.storage.model.Bucket bucketPb) {
+            return Bucket.fromPb(bucketPb);
+          }
+        }));
   }
 
   @Override
-  public Iterator list(String bucket, BlobIterOptions settings) {
-    // todo implement paging (with retries) with if limit is not given or > X
-    String delimiter = settings.recursive() ? options().pathDelimiter() : null;
-    return Iterators.transform(
-        storageRpc.list(bucket, settings.prefix(), delimiter, settings.cursor(),
-            settings.includeOlderVersions(), settings.maxResults()), Blob.FROM_PB_FUNCTION);
+  public ListResult list(final String bucket, BlobListOption... options) {
+    final Map optionsMap = optionMap(null, null, options);
+    Tuple> result = runWithRetries(
+        new Callable>>() {
+          @Override
+          public Tuple> call() {
+            return storageRpc.list(bucket, optionsMap);
+          }
+        }, retryParams, EXCEPTION_HANDLER);
+    return new ListResult<>(result.x(), Iterables.transform(result.y(),
+        new Function() {
+          @Override
+          public Blob apply(StorageObject storageObject) {
+            return Blob.fromPb(storageObject);
+          }
+        }));
   }
 
   @Override
   public Bucket update(Bucket bucket, BucketTargetOption... options) {
-    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(bucket, options);
     return Bucket.fromPb(runWithRetries(
         new Callable() {
           @Override
@@ -158,9 +174,8 @@ public com.google.api.services.storage.model.Bucket call() {
 
   @Override
   public Blob update(Blob blob, BlobTargetOption... options) {
-    Validator.checkBlobOptions(blob, options);
     final StorageObject storageObject = blob.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(blob, options);
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
@@ -171,9 +186,8 @@ public StorageObject call() {
 
   @Override
   public void delete(Bucket bucket, BucketSourceOption... options) {
-    Validator.checkBucketOptions(bucket, options);
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(bucket, options);
     runWithRetries(new Callable() {
       @Override
       public Void call() {
@@ -185,9 +199,8 @@ public Void call() {
 
   @Override
   public void delete(Blob blob, BlobSourceOption... options) {
-    Validator.checkBlobOptions(blob, options);
     final StorageObject storageObject = blob.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(blob, options);
     runWithRetries(new Callable() {
       @Override
       public Void call() {
@@ -202,11 +215,12 @@ public Blob compose(final ComposeRequest composeRequest) {
     final List sources =
         Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size());
     for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) {
-      sources.add(Blob.builder(composeRequest.sourceBucket(), sourceBlob.blob)
-          .generation(sourceBlob.generation).build().toPb());
+      sources.add(Blob.builder(composeRequest.target().bucket(), sourceBlob.name())
+          .generation(sourceBlob.generation()).build().toPb());
     }
     final StorageObject target = composeRequest.target().toPb();
-    final Map targetOptions = optionMap(composeRequest.targetOptions());
+    final Map targetOptions = optionMap(composeRequest.target().generation(),
+        composeRequest.target().metageneration(), composeRequest.targetOptions());
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
@@ -218,9 +232,12 @@ public StorageObject call() {
   @Override
   public Blob copy(CopyRequest copyRequest) {
     final StorageObject source = copyRequest.source().toPb();
-    final Map sourceOptions = optionMap(copyRequest.sourceOptions());
+    copyRequest.sourceOptions();
+    final Map sourceOptions = optionMap(copyRequest.source().generation(),
+        copyRequest.source().metageneration(), copyRequest.sourceOptions(), true);
     final StorageObject target = copyRequest.target().toPb();
-    final Map targetOptions = optionMap(copyRequest.targetOptions());
+    final Map targetOptions = optionMap(copyRequest.target().generation(),
+        copyRequest.target().metageneration(), copyRequest.targetOptions());
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
@@ -231,9 +248,8 @@ public StorageObject call() {
 
   @Override
   public byte[] load(Blob blob, BlobSourceOption... options) {
-    Validator.checkBlobOptions(blob, options);
     final StorageObject storageObject = blob.toPb();
-    final Map optionsMap = optionMap(options);
+    final Map optionsMap = optionMap(blob, options);
     return runWithRetries(new Callable() {
       @Override
       public byte[] call() {
@@ -244,29 +260,71 @@ public byte[] call() {
 
   @Override
   public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
-    Validator.checkBlobOptions(blob, options);
     // todo: Use retry helper on retriable failures
-    final Map optionsMap = optionMap(options);
+    // todo: consider changing lower level api to handle segments
+    final Map optionsMap = optionMap(blob, options);
     return storageRpc.reader(blob.toPb(), optionsMap);
   }
 
   @Override
   public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
-    Validator.checkBlobOptions(blob, options);
     // todo: Use retry helper on retriable failures
-    final Map optionsMap = optionMap(options);
+    // todo: consider changing lower level api to handle segments
+    final Map optionsMap = optionMap(blob, options);
     return storageRpc.writer(blob.toPb(), optionsMap);
   }
 
-  private static Map optionMap(Iterable options) {
-    ImmutableMap.Builder mapBuider = ImmutableMap.builder();
+  private Map optionMap(Long generation, Long metaGeneration,
+      Iterable options) {
+    return optionMap(generation, metaGeneration, options, false);
+  }
+
+  private Map optionMap(Long generation, Long metaGeneration,
+      Iterable options, boolean useAsSource) {
+    Map temp = Maps.newEnumMap(StorageRpc.Option.class);
     for (Option option : options) {
-      mapBuider.put(option.rpcOption(), option.value());
+      Object prev = temp.put(option.rpcOption(), option.value());
+      checkArgument(prev == null, "Duplicate option %s", option);
+    }
+    Boolean value = (Boolean) temp.remove(StorageRpc.Option.DELIMITER);
+    if (Boolean.TRUE.equals(value)) {
+      temp.put(StorageRpc.Option.DELIMITER, options().pathDelimiter());
     }
-    return mapBuider.build();
+    value = (Boolean) temp.remove(StorageRpc.Option.IF_GENERATION_MATCH);
+    if (value != null) {
+      checkArgument(generation != null, "missing generation value");
+      if (value) {
+        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_GENERATION_MATCH
+            : StorageRpc.Option.IF_GENERATION_MATCH, generation);
+      } else {
+        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH
+            : StorageRpc.Option.IF_GENERATION_NOT_MATCH, generation);
+      }
+    }
+    value = (Boolean) temp.remove(StorageRpc.Option.IF_METAGENERATION_MATCH);
+    if (value != null) {
+      checkArgument(metaGeneration != null, "missing metaGeneration value");
+      if (value) {
+        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH
+            : StorageRpc.Option.IF_METAGENERATION_MATCH, metaGeneration);
+      } else {
+        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH
+            : StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metaGeneration);
+      }
+    }
+    return ImmutableMap.copyOf(temp);
+  }
+
+  private Map optionMap(Long generation, Long metaGeneration,
+      Option... options) {
+    return optionMap(generation, metaGeneration, Arrays.asList(options));
+  }
+
+  private Map optionMap(Bucket bucket, Option... options) {
+    return optionMap(null, bucket.metageneration(), options);
   }
 
-  private static Map optionMap(Option... options) {
-    return optionMap(Arrays.asList(options));
+  private Map optionMap(Blob blob, Option... options) {
+    return optionMap(blob.generation(), blob.metageneration(), options);
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/Validator.java b/src/main/java/com/google/gcloud/storage/Validator.java
deleted file mode 100644
index 5590d67528cc..000000000000
--- a/src/main/java/com/google/gcloud/storage/Validator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.storage;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import java.util.Arrays;
-
-/**
- * Utility to validate Storage type/values.
- */
-public class Validator {
-
-  static void checkBlobOptions(Blob blob, Option... options) {
-    checkBlobOptions("requested", blob, Arrays.asList(options));
-  }
-
-  static void checkBlobOptions(String name, Blob blob, Iterable options) {
-    for (Option option : options) {
-      switch (option.rpcOption()) {
-        case IF_GENERATION_MATCH:
-          checkArgument(blob.generation() != 0, "%s blob is missing generation", name);
-          break;
-        case IF_METAGENERATION_MATCH:
-          checkArgument(blob.metageneration() != 0, "%s blob is missing metageneration", name);
-          break;
-      }
-    }
-  }
-
-  static void checkBucketOptions(Bucket bucket, Option... options) {
-    for (Option option : options) {
-      switch (option.rpcOption()) {
-        case IF_METAGENERATION_MATCH:
-          checkArgument(bucket.metageneration() != 0, "bucket is missing metageneration");
-          break;
-      }
-    }
-  }
-}

From 047bbebb8eb227fa77b1237b24a481477d72ea84 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 28 Apr 2015 20:26:27 -0700
Subject: [PATCH 184/732] add default content type to compose

---
 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 1d999d23d5ef..35ee9ec22a1a 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects()
          .delete(blob.getBucket(), blob.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      request.setDestination(target);
    }
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects()
          .delete(blob.getBucket(), blob.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      target.setContentType("binary/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file

From 83871026a7d17d89cf63b75803d60677ec1c8675 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 29 Apr 2015 17:49:13 -0700
Subject: [PATCH 185/732] work in progress

---
 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 35ee9ec22a1a..0c2ef339a092 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects()
          .delete(blob.getBucket(), blob.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      target.setContentType("binary/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects()
          .delete(blob.getBucket(), blob.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file

From 2fc739f589a0107b5488c643af5466b4e219efee Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Fri, 1 May 2015 14:43:18 -0700
Subject: [PATCH 186/732] adding batch

---
 .../gcloud/examples/StorageExample.java       | 39 +++++++++++++++++--
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../google/gcloud/storage/BatchRequest.java   | 34 ++++++++++++++++
 .../google/gcloud/storage/BatchResponse.java  | 23 +++++++++++
 .../google/gcloud/storage/StorageService.java |  8 +++-
 .../gcloud/storage/StorageServiceImpl.java    | 32 ++++++++++-----
 7 files changed, 122 insertions(+), 18 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/BatchRequest.java
 create mode 100644 src/main/java/com/google/gcloud/storage/BatchResponse.java

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index b06a99217b99..b6c0a74d0ab3 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -17,6 +17,7 @@
 package com.google.gcloud.examples;
 
 import com.google.gcloud.spi.StorageRpc.Tuple;
+import com.google.gcloud.storage.BatchRequest;
 import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.StorageService;
@@ -45,7 +46,7 @@
  * 
  • run using maven -
  * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
  * -Dexec.args="project_id list []| info [ []]| get  |
- *  upload   []| delete  |
+ *  upload   []| delete  +|
  *  cp    | compose  + "}
  * 
    • 
  * 
@@ -81,6 +82,26 @@ public String params() {
     }
   }
 
+  private static abstract class BlobsAction extends StorageAction {
+
+    @Override
+    Blob[] parse(String... args) {
+      if (args.length < 2) {
+        throw new IllegalArgumentException();
+      }
+      Blob[] blobs = new Blob[args.length - 1];
+      for (int i = 1; i < args.length; i++) {
+        blobs[i - 1] = Blob.of(args[0], args[i]);
+      }
+      return blobs;
+    }
+
+    @Override
+    public String params() {
+      return " +";
+    }
+  }
+
   private static class InfoAction extends BlobAction {
     @Override
     public void run(StorageService storage, Blob blob) {
@@ -105,10 +126,20 @@ public String params() {
     }
   }
 
-  private static class DeleteAction extends BlobAction {
+  private static class DeleteAction extends BlobsAction {
     @Override
-    public void run(StorageService storage, Blob blob) {
-      storage.delete(blob);
+    public void run(StorageService storage, Blob... blobs) {
+      if (blobs.length == 1) {
+        System.out.println("Calling delete");
+        System.out.println(storage.delete(blobs[0]));
+      } else {
+        BatchRequest batch = new BatchRequest();
+        for (Blob blob : blobs) {
+          batch.delete(blob);
+        }
+        System.out.println("Calling batch.delete");
+        storage.apply(batch);
+      }
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 0c2ef339a092..119d4ad1c698 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer).build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      GoogleJsonError details = ((GoogleJsonResponseException) exception).getDetails();
      boolean retryable = RETRYABLE_CODES.contains(details.getCode())
          || "InternalError".equals(details.getMessage());
      translated = new StorageServiceException(details.getCode(), details.getMessage(), retryable);
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void delete(StorageObject blob, Map options) {
    try {
      storage.objects()
          .delete(blob.getBucket(), blob.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.*;

import com.google.api.client.googleapis.batch.BatchRequest;
import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void batch(Iterable>> toDelete)
      throws StorageServiceException {
    BatchRequest batch = storage.batch();
    final Map failures = Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void storageObject, HttpHeaders responseHeaders) {
            // do nothing
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            failures.put(tuple.x(), translate(e));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 1140fc7469f8..c41ac53c6e41 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  void delete(Bucket bucket, Map options) throws StorageServiceException;

  void delete(StorageObject object, Map options) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  void batch(Iterable>> toDelete)
      throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java
new file mode 100644
index 000000000000..0f9e56f8a866
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import com.google.gcloud.storage.StorageService.BlobSourceOption;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Created by ozarov on 4/30/15.
+ */
+public class BatchRequest {
+
+  Map toDelete = new LinkedHashMap<>();
+
+  public void delete(Blob blob, BlobSourceOption... options) {
+    toDelete.put(blob, options);
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
new file mode 100644
index 000000000000..6fe0a14fd31e
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+/**
+ * Created by ozarov on 4/30/15.
+ */
+public class BatchResponse {
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 714343a96c5d..5933f6bdfeef 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -391,12 +391,13 @@ public static Builder builder() {
   /**
    * @throws StorageServiceException upon failure
    */
-  void delete(Bucket bucket, BucketSourceOption... options);
+  boolean delete(Bucket bucket, BucketSourceOption... options);
 
   /**
    * @throws StorageServiceException upon failure
    */
-  void delete(Blob blob, BlobSourceOption... options);
+  boolean delete(Blob blob, BlobSourceOption... options);
+
 
   /**
    * @throws StorageServiceException upon failure
@@ -414,6 +415,9 @@ public static Builder builder() {
    */
   byte[] load(Blob blob, BlobSourceOption... options);
 
+
+  BatchResponse apply(BatchRequest batchRequest);
+
   /**
    * @throws StorageServiceException upon failure
    */
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 5288c064e626..9dc4aa558441 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -185,27 +185,25 @@ public StorageObject call() {
   }
 
   @Override
-  public void delete(Bucket bucket, BucketSourceOption... options) {
+  public boolean delete(Bucket bucket, BucketSourceOption... options) {
     final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
     final Map optionsMap = optionMap(bucket, options);
-    runWithRetries(new Callable() {
+    return runWithRetries(new Callable() {
       @Override
-      public Void call() {
-        storageRpc.delete(bucketPb, optionsMap);
-        return null;
+      public Boolean call() {
+        return storageRpc.delete(bucketPb, optionsMap);
       }
     }, retryParams, EXCEPTION_HANDLER);
   }
 
   @Override
-  public void delete(Blob blob, BlobSourceOption... options) {
+  public boolean delete(Blob blob, BlobSourceOption... options) {
     final StorageObject storageObject = blob.toPb();
     final Map optionsMap = optionMap(blob, options);
-    runWithRetries(new Callable() {
+    return runWithRetries(new Callable() {
       @Override
-      public Void call() {
-        storageRpc.delete(storageObject, optionsMap);
-        return null;
+      public Boolean call() {
+        return storageRpc.delete(storageObject, optionsMap);
       }
     }, retryParams, EXCEPTION_HANDLER);
   }
@@ -258,6 +256,20 @@ public byte[] call() {
     }, retryParams, EXCEPTION_HANDLER);
   }
 
+  @Override
+  public BatchResponse apply(BatchRequest batchRequest) {
+    BatchResponse response = new BatchResponse();
+    List>> request =
+        Lists.newArrayListWithCapacity(batchRequest.toDelete.size());
+    for (Map.Entry entry : batchRequest.toDelete.entrySet()) {
+      Blob blob = entry.getKey();
+      Map optionsMap = optionMap(blob, entry.getValue());
+      request.add(Tuple.>of(blob.toPb(), optionsMap));
+    }
+    storageRpc.batch(request);
+    return response;
+  }
+
   @Override
   public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
     // todo: Use retry helper on retriable failures

From 075fc6c56115c00d19aa7f3f7f883533eba7a7de Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 1 May 2015 17:39:00 -0700
Subject: [PATCH 187/732] work in progress

---
 .../gcloud/examples/StorageExample.java       |  4 +-
 .../google/gcloud/storage/BatchRequest.java   | 59 +++++++++++--
 .../google/gcloud/storage/BatchResponse.java  | 83 ++++++++++++++++++-
 .../google/gcloud/storage/StorageService.java |  2 -
 .../gcloud/storage/StorageServiceImpl.java    | 13 ++-
 5 files changed, 146 insertions(+), 15 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index b6c0a74d0ab3..ee37b7e937d4 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -133,12 +133,12 @@ public void run(StorageService storage, Blob... blobs) {
         System.out.println("Calling delete");
         System.out.println(storage.delete(blobs[0]));
       } else {
-        BatchRequest batch = new BatchRequest();
+        BatchRequest.Builder batch = BatchRequest.builder();
         for (Blob blob : blobs) {
           batch.delete(blob);
         }
         System.out.println("Calling batch.delete");
-        storage.apply(batch);
+        storage.apply(batch.build());
       }
     }
   }
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 0f9e56f8a866..0056f354ac2b 100644
--- a/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -16,19 +16,68 @@
 
 package com.google.gcloud.storage;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.gcloud.storage.StorageService.BlobSourceOption;
+import com.google.gcloud.storage.StorageService.BlobTargetOption;
 
+import java.io.Serializable;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
 /**
- * Created by ozarov on 4/30/15.
+ * Google storage batch request.
  */
-public class BatchRequest {
+public class BatchRequest implements Serializable {
 
-  Map toDelete = new LinkedHashMap<>();
+  private static final long serialVersionUID = -1527992265939800345L;
 
-  public void delete(Blob blob, BlobSourceOption... options) {
-    toDelete.put(blob, options);
+  private Map toDelete;
+  private Map toUpdate;
+  private Map toGet;
+
+  public static class Builder {
+
+    private Map toDelete = new LinkedHashMap<>();
+    private Map toUpdate = new LinkedHashMap<>();
+    private Map toGet = new LinkedHashMap<>();
+
+    private Builder() {}
+
+    public void delete(Blob blob, BlobSourceOption... options) {
+      toDelete.put(blob, options);
+    }
+
+    public void update(Blob blob, BlobTargetOption... options) {
+      toUpdate.put(blob, options);
+    }
+
+    public void get(Blob blob, BlobSourceOption... options) {
+      toGet.put(blob, options);
+    }
+
+    public BatchRequest build() {
+      return new BatchRequest(this);
+    }
+  }
+
+  private BatchRequest(Builder builder) {
+    toDelete = ImmutableMap.copyOf(builder.toDelete);
+    toUpdate = ImmutableMap.copyOf(builder.toUpdate);
+  }
+
+  public Map toDelete() {
+    return toDelete;
+  }
+
+  public Map toUpdate() {
+    return toUpdate;
+  }
+
+  public Map toGet() {
+    return toGet;
+  }
+
+  public static Builder builder() {
+    return new Builder();
   }
 }
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index 6fe0a14fd31e..9c835b54fbee 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -16,8 +16,87 @@
 
 package com.google.gcloud.storage;
 
+import com.google.common.collect.ImmutableList;
+
+import java.io.Serializable;
+import java.util.List;
+
 /**
- * Created by ozarov on 4/30/15.
+ * Google Storage batch response.
  */
-public class BatchResponse {
+public class BatchResponse implements Serializable {
+
+  private List> deleteResult;
+  private List> updateResult;
+  private List> getResult;
+
+  public static class Result implements Serializable {
+
+    private static final long serialVersionUID = -1946539570170529094L;
+
+    private final T value;
+    private final StorageServiceException exception;
+
+    Result(T value) {
+      this.value = value;
+      this.exception = null;
+    }
+
+    Result(StorageServiceException exception) {
+      this.exception = exception;
+      this.value = null;
+    }
+
+
+    /**
+     * Returns the result.
+     *
+     * @throws StorageServiceException if failed
+     */
+    public T result() throws StorageServiceException {
+      return value;
+    }
+
+    /**
+     * Returns the failure or {@code null} if was successful.
+     */
+    public StorageServiceException failure() {
+      return exception;
+    }
+
+    /**
+     * Returns {@code true} if failed, {@code false} otherwise.
+     */
+    public boolean failed() {
+      return exception != null;
+    }
+  }
+
+  BatchResponse(List> deleteResult, List> updateResult,
+      List> getResult) {
+    this.deleteResult = ImmutableList.copyOf(deleteResult);
+    this.updateResult = ImmutableList.copyOf(updateResult);
+    this.getResult = ImmutableList.copyOf(getResult);
+  }
+
+  /**
+   * Returns the results for the delete operations using the request order.
+   */
+  public List> deletes() {
+    return deleteResult;
+  }
+
+  /**
+   * Returns the results for the update operations using the request order.
+   */
+  public List> updates() {
+    return updateResult;
+  }
+
+  /**
+   * Returns the results for the get operations using the request order.
+   */
+  public List> gets() {
+    return getResult;
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 5933f6bdfeef..71026fda6078 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -398,7 +398,6 @@ public static Builder builder() {
    */
   boolean delete(Blob blob, BlobSourceOption... options);
 
-
   /**
    * @throws StorageServiceException upon failure
    */
@@ -409,7 +408,6 @@ public static Builder builder() {
    */
   Blob copy(CopyRequest copyRequest);
 
-
   /**
    * @throws StorageServiceException upon failure
    */
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 9dc4aa558441..478df47949c9 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -34,6 +34,7 @@
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
 import java.util.Arrays;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
@@ -258,16 +259,20 @@ public byte[] call() {
 
   @Override
   public BatchResponse apply(BatchRequest batchRequest) {
-    BatchResponse response = new BatchResponse();
+    List> deletes = new LinkedList<>();
+    List> updates = new LinkedList<>();
+    List> gets = new LinkedList<>();
     List>> request =
-        Lists.newArrayListWithCapacity(batchRequest.toDelete.size());
-    for (Map.Entry entry : batchRequest.toDelete.entrySet()) {
+        Lists.newArrayListWithCapacity(batchRequest.toDelete().size());
+    for (Map.Entry entry : batchRequest.toDelete().entrySet()) {
       Blob blob = entry.getKey();
       Map optionsMap = optionMap(blob, entry.getValue());
       request.add(Tuple.>of(blob.toPb(), optionsMap));
     }
+    // todo: populate deletes, updates and gets (the latter 2 needs to be sent to RPC)
+    // todo: change example to use multi-update and multi-get
     storageRpc.batch(request);
-    return response;
+    return new BatchResponse(deletes, updates, gets);
   }
 
   @Override

From be58a4c85f2cf300ac902f1d362b6163e9709284 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sat, 2 May 2015 12:51:08 -0700
Subject: [PATCH 188/732] work in progress

---
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../gcloud/storage/StorageServiceImpl.java    | 37 ++++++++++++++++---
 3 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 119d4ad1c698..b31964d40c55 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.*;

import com.google.api.client.googleapis.batch.BatchRequest;
import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return storage.objects()
          .get(object.getBucket(), object.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return storage.objects()
          .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void batch(Iterable>> toDelete)
      throws StorageServiceException {
    BatchRequest batch = storage.batch();
    final Map failures = Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void storageObject, HttpHeaders responseHeaders) {
            // do nothing
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            failures.put(tuple.x(), translate(e));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.*;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index c41ac53c6e41..4b7600135f6c 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  void batch(Iterable>> toDelete)
      throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 478df47949c9..50c7bb039c7c 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -259,19 +259,44 @@ public byte[] call() {
 
   @Override
   public BatchResponse apply(BatchRequest batchRequest) {
-    List> deletes = new LinkedList<>();
-    List> updates = new LinkedList<>();
-    List> gets = new LinkedList<>();
-    List>> request =
+    List>> toDelete =
         Lists.newArrayListWithCapacity(batchRequest.toDelete().size());
     for (Map.Entry entry : batchRequest.toDelete().entrySet()) {
       Blob blob = entry.getKey();
       Map optionsMap = optionMap(blob, entry.getValue());
-      request.add(Tuple.>of(blob.toPb(), optionsMap));
+      toDelete.add(Tuple.>of(blob.toPb(), optionsMap));
+    }
+    List>> toUpdate =
+        Lists.newArrayListWithCapacity(batchRequest.toUpdate().size());
+    for (Map.Entry entry : batchRequest.toUpdate().entrySet()) {
+      Blob blob = entry.getKey();
+      Map optionsMap = optionMap(blob, entry.getValue());
+      toUpdate.add(Tuple.>of(blob.toPb(), optionsMap));
+    }
+    List>> toGet =
+        Lists.newArrayListWithCapacity(batchRequest.toGet().size());
+    for (Map.Entry entry : batchRequest.toGet().entrySet()) {
+      Blob blob = entry.getKey();
+      Map optionsMap = optionMap(blob, entry.getValue());
+      toGet.add(Tuple.>of(blob.toPb(), optionsMap));
     }
     // todo: populate deletes, updates and gets (the latter 2 needs to be sent to RPC)
     // todo: change example to use multi-update and multi-get
-    storageRpc.batch(request);
+    StorageRpc.BatchResponse response =
+        storageRpc.batch(new StorageRpc.BatchRequest(toDelete, toUpdate, toGet));
+    List> deletes = new LinkedList<>();
+    for (Blob blob : batchRequest.toDelete().keySet()) {
+      Tuple result = response.deletes.get(blob);
+      if (result.x() != null) {
+        deletes.add(new BatchResponse.Result<>(false));
+      } else {
+        deletes.add(new BatchResponse.Result(result.y()));
+      }
+    }
+
+    List> updates = new LinkedList<>();
+    List> gets = new LinkedList<>();
+
     return new BatchResponse(deletes, updates, gets);
   }
 

From ef0936177fc1df181e79e83f18a252894c6b0878 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 3 May 2015 17:17:04 -0700
Subject: [PATCH 189/732] complete batch

---
 .../gcloud/storage/StorageServiceImpl.java    | 36 +++++++++++--------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 50c7bb039c7c..45e5430c1505 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -21,6 +21,7 @@
 
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Function;
+import com.google.common.base.Functions;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
@@ -33,8 +34,8 @@
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
+import java.io.Serializable;
 import java.util.Arrays;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
@@ -264,7 +265,8 @@ public BatchResponse apply(BatchRequest batchRequest) {
     for (Map.Entry entry : batchRequest.toDelete().entrySet()) {
       Blob blob = entry.getKey();
       Map optionsMap = optionMap(blob, entry.getValue());
-      toDelete.add(Tuple.>of(blob.toPb(), optionsMap));
+      StorageObject storageObject = blob.toPb();
+      toDelete.add(Tuple.>of(storageObject, optionsMap));
     }
     List>> toUpdate =
         Lists.newArrayListWithCapacity(batchRequest.toUpdate().size());
@@ -280,24 +282,30 @@ public BatchResponse apply(BatchRequest batchRequest) {
       Map optionsMap = optionMap(blob, entry.getValue());
       toGet.add(Tuple.>of(blob.toPb(), optionsMap));
     }
-    // todo: populate deletes, updates and gets (the latter 2 needs to be sent to RPC)
-    // todo: change example to use multi-update and multi-get
     StorageRpc.BatchResponse response =
         storageRpc.batch(new StorageRpc.BatchRequest(toDelete, toUpdate, toGet));
-    List> deletes = new LinkedList<>();
-    for (Blob blob : batchRequest.toDelete().keySet()) {
-      Tuple result = response.deletes.get(blob);
+    List> deletes = transformBatchResult(
+        toDelete, response.deletes, Functions.identity());
+    List> updates = transformBatchResult(
+        toUpdate, response.updates, Blob.FROM_PB_FUNCTION);
+    List> gets = transformBatchResult(
+        toGet, response.gets, Blob.FROM_PB_FUNCTION);
+    return new BatchResponse(deletes, updates, gets);
+  }
+
+  private  List> transformBatchResult(
+      Iterable>> request,
+      Map> results, Function transform) {
+    List> response = Lists.newArrayListWithCapacity(results.size());
+    for (Tuple tuple : request) {
+      Tuple result = results.get(tuple.x());
       if (result.x() != null) {
-        deletes.add(new BatchResponse.Result<>(false));
+        response.add(new BatchResponse.Result<>(transform.apply(result.x())));
       } else {
-        deletes.add(new BatchResponse.Result(result.y()));
+        response.add(new BatchResponse.Result(result.y()));
       }
     }
-
-    List> updates = new LinkedList<>();
-    List> gets = new LinkedList<>();
-
-    return new BatchResponse(deletes, updates, gets);
+    return response;
   }
 
   @Override

From a24c7bc15e30ce37dd47936e409df9a519cbd923 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 3 May 2015 17:50:12 -0700
Subject: [PATCH 190/732] add multi info to example

---
 .../gcloud/examples/StorageExample.java       | 33 ++++++++++++-------
 .../google/gcloud/storage/BatchRequest.java   |  7 ++--
 .../google/gcloud/storage/BatchResponse.java  |  6 ++++
 3 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index ee37b7e937d4..00d16b19f713 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -18,11 +18,12 @@
 
 import com.google.gcloud.spi.StorageRpc.Tuple;
 import com.google.gcloud.storage.BatchRequest;
+import com.google.gcloud.storage.BatchResponse;
 import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.StorageService;
-import com.google.gcloud.storage.StorageService.CopyRequest;
 import com.google.gcloud.storage.StorageService.ComposeRequest;
+import com.google.gcloud.storage.StorageService.CopyRequest;
 import com.google.gcloud.storage.StorageServiceFactory;
 import com.google.gcloud.storage.StorageServiceOptions;
 
@@ -102,27 +103,36 @@ public String params() {
     }
   }
 
-  private static class InfoAction extends BlobAction {
+  private static class InfoAction extends BlobsAction {
     @Override
-    public void run(StorageService storage, Blob blob) {
-      if (blob.name().isEmpty()) {
-        System.out.println(storage.get(Bucket.of(blob.bucket())));
+    public void run(StorageService storage, Blob... blobs) {
+      if (blobs.length == 1) {
+        if (blobs[0].name().isEmpty()) {
+          System.out.println(storage.get(Bucket.of(blobs[0].bucket())));
+        } else {
+          System.out.println(storage.get(blobs[0]));
+        }
       } else {
-        System.out.println(storage.get(blob));
+        BatchRequest.Builder batch = BatchRequest.builder();
+        for (Blob blob : blobs) {
+          batch.get(blob);
+        }
+        BatchResponse response = storage.apply(batch.build());
+        System.out.println(response.gets());
       }
     }
 
     @Override
-    Blob parse(String... args) {
+    Blob[] parse(String... args) {
       if (args.length < 2) {
-        return Blob.of(args[0], "");
+        return new Blob[] {Blob.of(args[0], "")};
       }
       return super.parse(args);
     }
 
     @Override
     public String params() {
-      return " []";
+      return " [+]";
     }
   }
 
@@ -130,15 +140,14 @@ private static class DeleteAction extends BlobsAction {
     @Override
     public void run(StorageService storage, Blob... blobs) {
       if (blobs.length == 1) {
-        System.out.println("Calling delete");
         System.out.println(storage.delete(blobs[0]));
       } else {
         BatchRequest.Builder batch = BatchRequest.builder();
         for (Blob blob : blobs) {
           batch.delete(blob);
         }
-        System.out.println("Calling batch.delete");
-        storage.apply(batch.build());
+        BatchResponse response = storage.apply(batch.build());
+        System.out.println(response.deletes());
       }
     }
   }
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 0056f354ac2b..4f27abd89547 100644
--- a/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -31,9 +31,9 @@ public class BatchRequest implements Serializable {
 
   private static final long serialVersionUID = -1527992265939800345L;
 
-  private Map toDelete;
-  private Map toUpdate;
-  private Map toGet;
+  private final Map toDelete;
+  private final Map toUpdate;
+  private final Map toGet;
 
   public static class Builder {
 
@@ -63,6 +63,7 @@ public BatchRequest build() {
   private BatchRequest(Builder builder) {
     toDelete = ImmutableMap.copyOf(builder.toDelete);
     toUpdate = ImmutableMap.copyOf(builder.toUpdate);
+    toGet = ImmutableMap.copyOf(builder.toGet);
   }
 
   public Map toDelete() {
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index 9c835b54fbee..bef428c37a74 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.storage;
 
+import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 
 import java.io.Serializable;
@@ -70,6 +71,11 @@ public StorageServiceException failure() {
     public boolean failed() {
       return exception != null;
     }
+
+    @Override
+    public String toString() {
+      return MoreObjects.firstNonNull(value, exception).toString();
+    }
   }
 
   BatchResponse(List> deleteResult, List> updateResult,

From d9b994bccf49a33707fa15b3ffc4020bcf935913 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 3 May 2015 18:01:43 -0700
Subject: [PATCH 191/732] work in progress

---
 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index b31964d40c55..4bf7b8ec647d 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.*;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.*;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement (consider passing a range from, to) and return ByteBuffer
    try {
      storage.objects().get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      //.getMediaHttpDownloader().setContentRange()
    } catch (IOException ex) {
      throw translate(ex);
    }
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement (consider passing headers, resumable-write, from
    // and get byte buffer
    return null;
  }
}
\ No newline at end of file

From 648ae8d389f3a7b72153b2724b8ed2ef3d0f780b Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 4 May 2015 12:31:40 -0700
Subject: [PATCH 192/732] find project-id from env

---
 .../com/google/gcloud/ServiceOptions.java     | 39 +++++++++++++++----
 .../gcloud/examples/StorageExample.java       | 19 ++++++---
 2 files changed, 45 insertions(+), 13 deletions(-)

diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index ea3d37f80b9e..082caa7eb6ee 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -27,14 +27,19 @@
 import com.google.gcloud.spi.ServiceRpcFactory;
 
 import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Serializable;
 import java.lang.reflect.Method;
+import java.net.HttpURLConnection;
 import java.net.URL;
-import java.net.URLConnection;
 import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public abstract class ServiceOptions> implements Serializable {
 
@@ -168,16 +173,36 @@ protected static String appEngineAppId() {
   protected static String googleCloudProjectId() {
     try {
       URL url = new URL("http://metadata/computeMetadata/v1/project/project-id");
-      URLConnection connection = url.openConnection();
+      HttpURLConnection connection = (HttpURLConnection) url.openConnection();
       connection.setRequestProperty("X-Google-Metadata-Request", "True");
-      try (BufferedReader reader =
-          new BufferedReader(new InputStreamReader(connection.getInputStream(), UTF_8))) {
-        return reader.readLine();
+      InputStream input = connection.getInputStream();
+      if (connection.getResponseCode() == 200) {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) {
+          return reader.readLine();
+        }
       }
     } catch (IOException ignore) {
-      // return null if can't determine
-      return null;
+      // ignore
+    }
+    String configDir = System.getenv("CLOUDSDK_CONFIG");
+    if (configDir == null) {
+      configDir = new File(System.getProperty("user.home"), "/.config/gcloud/").getPath();
+    }
+    try (BufferedReader reader =
+        new BufferedReader(new FileReader(new File(configDir, "properties")))) {
+      String line;
+      Pattern pattern = Pattern.compile("^\\s*project\\s*=\\s*(.*?)\\s*$");
+      while((line = reader.readLine()) != null) {
+        Matcher matcher = pattern.matcher(line);
+        if (matcher.matches()) {
+          return matcher.group(1);
+        }
+      }
+    } catch (IOException ex) {
+      // ignore
     }
+    // return null if can't determine
+    return null;
   }
 
   protected static String getAppEngineProjectId() {
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 00d16b19f713..c93d7d0885a4 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -46,7 +46,7 @@
  * 
  • compile using maven - {@code mvn compile}
    • 
  * 
  • run using maven -
  * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
- * -Dexec.args="project_id list []| info [ []]| get  |
+ * -Dexec.args="[] list []| info [ []]| get  |
  *  upload   []| delete  +|
  *  cp    | compose  + "}
  * 
    • 
@@ -300,20 +300,27 @@ public static void printUsage() {
 
   @SuppressWarnings("unchecked")
   public static void main(String... args) throws Exception {
-    if (args.length < 2) {
+    if (args.length < 1) {
       System.out.println("Missing required project id and action");
       printUsage();
       return;
     }
-    StorageAction action = ACTIONS.get(args[1]);
+    StorageServiceOptions.Builder optionsBuilder = StorageServiceOptions.builder();
+    StorageAction action;
+    if (args.length >= 2 && !ACTIONS.containsKey(args[0])) {
+      optionsBuilder.project(args[0]);
+      action = ACTIONS.get(args[1]);
+      args = Arrays.copyOfRange(args, 2, args.length);
+    } else {
+      action = ACTIONS.get(args[0]);
+      args = Arrays.copyOfRange(args, 1, args.length);
+    }
     if (action == null) {
       System.out.println("Unrecognized action '" + args[1] + "'");
       printUsage();
       return;
     }
-    StorageServiceOptions options = StorageServiceOptions.builder().project(args[0]).build();
-    StorageService storage = StorageServiceFactory.instance().get(options);
-    args = args.length > 2 ? Arrays.copyOfRange(args, 2, args.length) : new String[] {};
+    StorageService storage = StorageServiceFactory.instance().get(optionsBuilder.build());
     Object request;
     try {
       request = action.parse(args);

From dff49797ada1f85eb592eeefae2affef3ef6e193 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 4 May 2015 13:30:14 -0700
Subject: [PATCH 193/732] update ini hanlding

---
 .../java/com/google/gcloud/ServiceOptions.java  | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 082caa7eb6ee..0b03e7951e86 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -191,11 +191,22 @@ protected static String googleCloudProjectId() {
     try (BufferedReader reader =
         new BufferedReader(new FileReader(new File(configDir, "properties")))) {
       String line;
-      Pattern pattern = Pattern.compile("^\\s*project\\s*=\\s*(.*?)\\s*$");
+      String section = null;
+      Pattern projectPattern = Pattern.compile("^project\\s*=\\s*(.*)$");
+      Pattern sectionPattern = Pattern.compile("^\\[(.*)\\]$");
       while((line = reader.readLine()) != null) {
-        Matcher matcher = pattern.matcher(line);
+        if (line.isEmpty() || line.startsWith(";")) {
+          continue;
+        }
+        line = line.trim();
+        Matcher matcher = sectionPattern.matcher(line);
         if (matcher.matches()) {
-          return matcher.group(1);
+          section = matcher.group(1);
+        } else if (section == null || section.equals("core")) {
+          matcher = projectPattern.matcher(line);
+          if (matcher.matches()) {
+            return matcher.group(1);
+          }
         }
       }
     } catch (IOException ex) {

From c1db9b21b5871ab7c5af96c1e1156d75dd410b58 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 4 May 2015 16:51:49 -0700
Subject: [PATCH 194/732] implement Serializable reader

---
 .../gcloud/examples/StorageExample.java       |  24 ++--
 .../google/gcloud/spi/DefaultStorageRpc.java  |   2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../gcloud/storage/BlobReadChannel.java       |   3 +-
 .../gcloud/storage/StorageServiceImpl.java    | 111 +++++++++++++++++-
 .../gcloud/storage/StorageServiceOptions.java |   1 +
 6 files changed, 129 insertions(+), 14 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index c93d7d0885a4..5fa003c5672c 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,10 +16,13 @@
 
 package com.google.gcloud.examples;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.google.gcloud.spi.StorageRpc.Tuple;
 import com.google.gcloud.storage.BatchRequest;
 import com.google.gcloud.storage.BatchResponse;
 import com.google.gcloud.storage.Blob;
+import com.google.gcloud.storage.BlobReadChannel;
 import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.StorageService;
 import com.google.gcloud.storage.StorageService.ComposeRequest;
@@ -27,7 +30,8 @@
 import com.google.gcloud.storage.StorageServiceFactory;
 import com.google.gcloud.storage.StorageServiceOptions;
 
-import java.nio.charset.StandardCharsets;
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -214,17 +218,23 @@ public String params() {
 
   private static class GetAction extends BlobAction {
     @Override
-    public void run(StorageService storage, Blob blob) {
+    public void run(StorageService storage, Blob blob) throws IOException {
       blob = storage.get(blob);
       if (blob == null) {
         System.out.println("No such object");
         return;
       }
-      if (blob.size() < 1_000_000) {
-        System.out.println(new String(storage.load(blob), StandardCharsets.UTF_8));
+      if (blob.size() < 1024) {
+        System.out.println(new String(storage.load(blob), UTF_8));
       } else {
-        // todo: download via streaming API
-        throw new IllegalArgumentException("file is too big");
+        try (BlobReadChannel reader = storage.reader(blob)) {
+          ByteBuffer bytes = ByteBuffer.allocate(64 * 1024);
+          while (reader.read(bytes) > 0) {
+            bytes.flip();
+            System.out.print(UTF_8.decode(bytes));
+            bytes.clear();
+          }
+        }
       }
     }
   }
@@ -294,7 +304,7 @@ public static void printUsage() {
         actionAndParams.append(' ').append(param);
       }
     }
-    System.out.printf("Usage: %s   *%s%n",
+    System.out.printf("Usage: %s [] operation *%s%n",
         StorageExample.class.getSimpleName(), actionAndParams);
   }
 
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 4bf7b8ec647d..0f45f06830b7 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.*;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException {
    // todo: implement (consider passing a range from, to) and return ByteBuffer
    try {
      storage.objects().get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      //.getMediaHttpDownloader().setContentRange()
    } catch (IOException ex) {
      throw translate(ex);
    }
    return null;
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement (consider passing headers, resumable-write, from
    // and get byte buffer
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      downloader.setContentRange(position, position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement (consider passing headers, resumable-write, from
    // and get byte buffer
    return null;
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 4b7600135f6c..14adcb6b502e 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.BlobReadChannel;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  BlobReadChannel reader(StorageObject from, Map options)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
index 32b68b3e97f6..89f3420a2a28 100644
--- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
@@ -17,6 +17,7 @@
 package com.google.gcloud.storage;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.io.Serializable;
 import java.nio.channels.ReadableByteChannel;
 
@@ -37,5 +38,5 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos
   @Override
   void close();
 
-  void seek(int position);
+  void seek(int position) throws IOException;
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 45e5430c1505..88c3a8df126b 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -34,7 +34,11 @@
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.io.Serializable;
+import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -308,12 +312,111 @@ private  List> transformBatch
     return response;
   }
 
+  private static class BlobReadChannelImpl implements BlobReadChannel {
+
+    private final StorageServiceOptions serviceOptions;
+    private final Blob blob;
+    private final Map requestOptions;
+    private int position;
+    private boolean isOpen;
+    private boolean endOfStream;
+
+    private transient StorageRpc storageRpc;
+    private transient RetryParams retryParams;
+    private transient StorageObject storageObject;
+    private transient int bufferPos;
+    private transient byte[] buffer;
+
+    BlobReadChannelImpl(StorageServiceOptions serviceOptions, Blob blob,
+        Map requestOptions) {
+      this.serviceOptions = serviceOptions;
+      this.blob = blob;
+      this.requestOptions = requestOptions;
+      isOpen = true;
+      initTransients();
+    }
+
+    private void writeObject(ObjectOutputStream out) throws IOException {
+      if (buffer != null) {
+        position += bufferPos;
+        buffer = null;
+        bufferPos = 0;
+        endOfStream = false;
+      }
+      out.defaultWriteObject();
+    }
+
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+      in.defaultReadObject();
+      initTransients();
+    }
+
+    private void initTransients() {
+      storageRpc = serviceOptions.storageRpc();
+      retryParams = MoreObjects.firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries());
+      storageObject = blob.toPb();
+    }
+
+    @Override
+    public boolean isOpen() {
+      return isOpen;
+    }
+
+    @Override
+    public void close() {
+      if (isOpen) {
+        buffer = null;
+        isOpen = false;
+      }
+    }
+
+    private void validateOpen() throws IOException {
+      if (!isOpen) {
+        throw new IOException("stream is closed");
+      }
+    }
+
+    @Override
+    public void seek(int position) throws IOException {
+      validateOpen();
+      throw new UnsupportedOperationException("not supported yet");
+      // todo: implement
+    }
+
+    @Override
+    public int read(ByteBuffer byteBuffer) throws IOException {
+      validateOpen();
+      if (buffer == null) {
+        if (endOfStream) {
+          return -1;
+        }
+        final int toRead = Math.max(byteBuffer.remaining(), 256 * 1024);
+        buffer = runWithRetries(new Callable() {
+          @Override
+          public byte[] call() {
+            return storageRpc.read(storageObject, requestOptions, position, toRead);
+          }
+        }, retryParams, EXCEPTION_HANDLER);
+        if (toRead > buffer.length) {
+          endOfStream = true;
+        }
+      }
+      int toWrite = Math.min(buffer.length - bufferPos, byteBuffer.remaining());
+      byteBuffer.put(buffer, bufferPos, toWrite);
+      bufferPos += toWrite;
+      if (bufferPos >= buffer.length) {
+        position += buffer.length;
+        buffer = null;
+        bufferPos = 0;
+      }
+      return toWrite;
+    }
+  }
+
   @Override
   public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
-    // todo: Use retry helper on retriable failures
-    // todo: consider changing lower level api to handle segments
-    final Map optionsMap = optionMap(blob, options);
-    return storageRpc.reader(blob.toPb(), optionsMap);
+    Map optionsMap = optionMap(blob, options);
+    return new BlobReadChannelImpl(options(), blob, optionsMap);
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index a08edeee7819..ed16dd3863a6 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -70,6 +70,7 @@ private StorageServiceOptions(Builder builder) {
     pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
     project = builder.project != null ? builder.project : defaultProject();
     Preconditions.checkArgument(project != null, "Missing required project id");
+    // todo: consider providing read-timeout
   }
 
   @Override

From bf089292e2926f32ec3a38153a0aaef14ff2752d Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 4 May 2015 17:35:16 -0700
Subject: [PATCH 195/732] work in progress

---
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../gcloud/storage/StorageServiceImpl.java    | 44 ++++++++++++++++---
 3 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 0f45f06830b7..4bf69d1dba70 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      downloader.setContentRange(position, position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException {
    // todo: implement (consider passing headers, resumable-write, from
    // and get byte buffer
    return null;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      downloader.setContentRange(position, position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last)
      throws StorageServiceException {
    return;
  }

  @Override
  public String write(byte[] bytes, StorageObject to, Map options)
      throws StorageServiceException {
    String type = MoreObjects.firstNonNull(to.getContentType(), "application/octet-stream");
    try {
      Insert req = storage.objects().insert(to.getBucket(), to, new ByteArrayContent(type, bytes));
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options));
      req.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options));
      req.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options));
      req.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      System.out.println("KOKO.request: " + req.getRequestMethod());
      System.out.println("KOKO.request.autherization: " + req.getRequestHeaders().getAuthorization());
      System.out.println("KOKO.request.authToken: " + req.getOauthToken());
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 14adcb6b502e..ddfdd311e386 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.BlobWriteChannel;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException;

  BlobWriteChannel writer(StorageObject to, Map options)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException;

  String write(byte[] bytes, StorageObject to, Map options)
      throws StorageServiceException;

  void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 88c3a8df126b..56e4293dd9ad 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -75,6 +75,10 @@ public RetryResult beforeEval(Exception exception) {
     storageRpc = options.storageRpc();
     retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
     // todo: replace nulls with Value.asNull (per toPb)
+    // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors
+    // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite
+    // todo: provide signed urls - https://cloud.google.com/storage/docs/access-control#Signed-URLs
+    // todo: check if we need to expose https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert vs using bucket update/patch
   }
 
   @Override
@@ -379,8 +383,10 @@ private void validateOpen() throws IOException {
     @Override
     public void seek(int position) throws IOException {
       validateOpen();
-      throw new UnsupportedOperationException("not supported yet");
-      // todo: implement
+      this.position = position;
+      buffer = null;
+      bufferPos = 0;
+      endOfStream = false;
     }
 
     @Override
@@ -419,12 +425,40 @@ public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
     return new BlobReadChannelImpl(options(), blob, optionsMap);
   }
 
+  private static class BlobWriterChannelImpl implements BlobWriteChannel {
+
+    private final StorageServiceOptions options;
+    private final Blob blob;
+    private final Map optionsMap;
+
+    public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
+        Map optionsMap) {
+      this.options = options;
+      this.blob = blob;
+      this.optionsMap = optionsMap;
+    }
+
+    @Override
+    public int write(ByteBuffer byteBuffer) throws IOException {
+      // todo: Use retry helper on retriable failures
+      return 0;
+    }
+
+    @Override
+    public boolean isOpen() {
+      return false;
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+  }
+
   @Override
   public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
-    // todo: Use retry helper on retriable failures
-    // todo: consider changing lower level api to handle segments
     final Map optionsMap = optionMap(blob, options);
-    return storageRpc.writer(blob.toPb(), optionsMap);
+    return new BlobWriterChannelImpl(options(), blob, optionsMap);
   }
 
   private Map optionMap(Long generation, Long metaGeneration,

From dc253c4f2798aac21c145f7bbe68fab93b438af7 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 5 May 2015 15:37:59 -0700
Subject: [PATCH 196/732] work in progress

---
 .../gcloud/datastore/DatastoreServiceOptions.java    |  4 ++++
 .../com/google/gcloud/examples/StorageExample.java   | 10 ++++++++++
 .../com/google/gcloud/spi/DefaultStorageRpc.java     |  2 +-
 src/main/java/com/google/gcloud/spi/StorageRpc.java  |  2 +-
 .../google/gcloud/storage/StorageServiceImpl.java    | 12 +++++++++++-
 .../google/gcloud/storage/StorageServiceOptions.java |  4 ++++
 6 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 8d5f062fd548..b1b1ee31638c 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -199,6 +199,10 @@ DatastoreRpc datastoreRpc() {
     return ServiceRpcProvider.datastore(this);
   }
 
+  public static DatastoreServiceOptions defaultIntance() {
+    return builder().build();
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 5fa003c5672c..0f9650a63bcc 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -23,6 +23,7 @@
 import com.google.gcloud.storage.BatchResponse;
 import com.google.gcloud.storage.Blob;
 import com.google.gcloud.storage.BlobReadChannel;
+import com.google.gcloud.storage.BlobWriteChannel;
 import com.google.gcloud.storage.Bucket;
 import com.google.gcloud.storage.StorageService;
 import com.google.gcloud.storage.StorageService.ComposeRequest;
@@ -311,6 +312,15 @@ public static void printUsage() {
   @SuppressWarnings("unchecked")
   public static void main(String... args) throws Exception {
     if (args.length < 1) {
+
+      System.out.println("KOKO.start -------------------------------");
+      StorageService storage =
+          StorageServiceFactory.instance().get(StorageServiceOptions.defaultInstnace());
+      BlobWriteChannel writer = storage
+          .writer(Blob.of("ozarov-javamrsample.appspot.com", "bla"));
+      writer.write(ByteBuffer.wrap("hello world".getBytes()));
+      writer.close();
+      System.out.println("KOKO.end -------------------------------");
       System.out.println("Missing required project id and action");
       printUsage();
       return;
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 4bf69d1dba70..938ab24c1ce8 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() != null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      downloader.setContentRange(position, position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last)
      throws StorageServiceException {
    return;
  }

  @Override
  public String write(byte[] bytes, StorageObject to, Map options)
      throws StorageServiceException {
    String type = MoreObjects.firstNonNull(to.getContentType(), "application/octet-stream");
    try {
      Insert req = storage.objects().insert(to.getBucket(), to, new ByteArrayContent(type, bytes));
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options));
      req.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options));
      req.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options));
      req.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      System.out.println("KOKO.request: " + req.getRequestMethod());
      System.out.println("KOKO.request.autherization: " + req.getRequestHeaders().getAuthorization());
      System.out.println("KOKO.request.authToken: " + req.getOauthToken());
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      downloader.setContentRange(position, position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, StorageObject to, String uploadId, int position, boolean last)
      throws StorageServiceException {
    return;
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    String type = MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream");
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      try {
        GenericUrl url = req.buildHttpRequest().getUrl();
        String scheme = url.getScheme();
        String host = url.getHost();
        String path = "/upload" + url.getRawPath();
        url = new GenericUrl(scheme + "://" + host + path);
        url.set("uploadType", "resumable");
        url.set("name", object.getName());
        for (Option option : options.keySet()) {
          Object value = option.get(options);
          if (value != null) {
            url.set(option.name(), value.toString());
          }
        }

        HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, null);
        httpRequest.getHeaders().setContentType(type);
        httpRequest.getHeaders().setContentLength(0L);
        System.out.println("koko.url: " + httpRequest.getUrl());
        System.out.println("koko.contentType:" + httpRequest.getHeaders().getContentType());
        System.out.println("koko.contentLength:" + httpRequest.getHeaders().getContentLength());

        httpRequest.setLoggingEnabled(true);
        httpRequest.setCurlLoggingEnabled(true);
        HttpResponse response = httpRequest.execute();
        System.out.println("koko.response: " + response.toString());
        System.out.println("koko.response.code: " + response.getStatusCode());
        System.out.println("koko.response.message: " + response.getStatusMessage());
        System.out.println("koko.response.content: " + response.getContentCharset());
      } catch (Exception ex) {
        ex.printStackTrace();
      }
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index ddfdd311e386..5bf96a6ea824 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException;

  String write(byte[] bytes, StorageObject to, Map options)
      throws StorageServiceException;

  void append(byte[] bytes, StorageObject to, String uploadId, int position, boolean last)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(byte[] bytes, StorageObject object, String uploadId, int position, boolean last)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 56e4293dd9ad..bd2ef247d103 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -430,18 +430,28 @@ private static class BlobWriterChannelImpl implements BlobWriteChannel {
     private final StorageServiceOptions options;
     private final Blob blob;
     private final Map optionsMap;
+    private final String uploadId;
+    private int position;
 
     public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
         Map optionsMap) {
       this.options = options;
       this.blob = blob;
       this.optionsMap = optionsMap;
+      System.out.println("Koko. BlobWriterChannelImpl.init");
+      uploadId = options.storageRpc().open(blob.toPb(), optionsMap);
     }
 
     @Override
     public int write(ByteBuffer byteBuffer) throws IOException {
+      int size = Math.min(byteBuffer.remaining(), 1024 * 1024);
+      byte[] bytes = new byte[size];
+      byteBuffer.get(bytes);
       // todo: Use retry helper on retriable failures
-      return 0;
+      System.out.println("Koko. BlobWriterChannelImpl.write");
+      options.storageRpc().write(bytes, blob.toPb(), uploadId, position, false);
+      position += size;
+      return size;
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index ed16dd3863a6..24dea9c91788 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -121,6 +121,10 @@ private static String defaultProject() {
     return projectId != null ? projectId : googleCloudProjectId();
   }
 
+  public static StorageServiceOptions defaultInstnace() {
+    return builder().build();
+  }
+
   public static Builder builder() {
     return new Builder();
   }

From 5d59576c30ab8a14519aa072adc485ba553bf002 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 5 May 2015 17:41:39 -0700
Subject: [PATCH 197/732] work in progress

---
 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java      | 2 +-
 src/main/java/com/google/gcloud/spi/StorageRpc.java             | 2 +-
 src/main/java/com/google/gcloud/storage/StorageServiceImpl.java | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 938ab24c1ce8..71b94384bbb2 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      downloader.setContentRange(position, position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, StorageObject to, String uploadId, int position, boolean last)
      throws StorageServiceException {
    return;
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    String type = MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream");
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      try {
        GenericUrl url = req.buildHttpRequest().getUrl();
        String scheme = url.getScheme();
        String host = url.getHost();
        String path = "/upload" + url.getRawPath();
        url = new GenericUrl(scheme + "://" + host + path);
        url.set("uploadType", "resumable");
        url.set("name", object.getName());
        for (Option option : options.keySet()) {
          Object value = option.get(options);
          if (value != null) {
            url.set(option.name(), value.toString());
          }
        }

        HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url, null);
        httpRequest.getHeaders().setContentType(type);
        httpRequest.getHeaders().setContentLength(0L);
        System.out.println("koko.url: " + httpRequest.getUrl());
        System.out.println("koko.contentType:" + httpRequest.getHeaders().getContentType());
        System.out.println("koko.contentLength:" + httpRequest.getHeaders().getContentLength());

        httpRequest.setLoggingEnabled(true);
        httpRequest.setCurlLoggingEnabled(true);
        HttpResponse response = httpRequest.execute();
        System.out.println("koko.response: " + response.toString());
        System.out.println("koko.response.code: " + response.getStatusCode());
        System.out.println("koko.response.message: " + response.getStatusMessage());
        System.out.println("koko.response.content: " + response.getContentCharset());
      } catch (Exception ex) {
        ex.printStackTrace();
      }
      return null;
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last)
      throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, bytes));
      long limit = position + bytes.length;
      String range = position + "-" + (bytes.length - 1);
      httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*"));
      throw new IOException("bla");
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index 5bf96a6ea824..e3e28c2dc56f 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, int position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(byte[] bytes, StorageObject object, String uploadId, int position, boolean last)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(byte[] bytes, StorageObject object, String uploadId, long position, boolean last)
      throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index bd2ef247d103..e84f9c2ff447 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -440,6 +440,7 @@ public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
       this.optionsMap = optionsMap;
       System.out.println("Koko. BlobWriterChannelImpl.init");
       uploadId = options.storageRpc().open(blob.toPb(), optionsMap);
+      System.out.println("Koko. BlobWriterChannelImpl.init.uploadId: " + uploadId);
     }
 
     @Override

From 77d5baf65bfa5abde120f4fe2c03f6aa5318ae28 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 5 May 2015 20:35:15 -0700
Subject: [PATCH 198/732] work in progress

---
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../gcloud/storage/StorageServiceImpl.java    | 54 ++++++++++++++++---
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 71b94384bbb2..502979953221 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last)
      throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, bytes));
      long limit = position + bytes.length;
      String range = position + "-" + (bytes.length - 1);
      httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*"));
      throw new IOException("bla");
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last)
      throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, bytes));
      long limit = position + bytes.length;
      String range = position + "-" + (bytes.length - 1);
      httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*"));
      HttpResponse response = httpRequest.execute();
      int code = response.getStatusCode();
      if (!last && code != 308 || last && (code != 200 || code != 201)) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index e84f9c2ff447..bfe1d78f8e03 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -427,25 +427,59 @@ public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
 
   private static class BlobWriterChannelImpl implements BlobWriteChannel {
 
+    private static final int CHUNK_SIZE = 256 * 1024;
+    private static final int MIN_BUFFER_SIZE = CHUNK_SIZE * 4;
+
     private final StorageServiceOptions options;
     private final Blob blob;
-    private final Map optionsMap;
     private final String uploadId;
     private int position;
+    private byte[] buffer = new byte[MIN_BUFFER_SIZE];
+    private int bufferLimit;
+    private boolean isOpen;
+
+    private transient StorageRpc storageRpc;
+    private transient RetryParams retryParams;
+    private transient StorageObject storageObject;
 
     public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
         Map optionsMap) {
       this.options = options;
       this.blob = blob;
-      this.optionsMap = optionsMap;
-      System.out.println("Koko. BlobWriterChannelImpl.init");
-      uploadId = options.storageRpc().open(blob.toPb(), optionsMap);
-      System.out.println("Koko. BlobWriterChannelImpl.init.uploadId: " + uploadId);
+      initTransients();
+      uploadId = options.storageRpc().open(storageObject, optionsMap);
+    }
+
+    private void writeObject(ObjectOutputStream out) throws IOException {
+      out.defaultWriteObject();
+    }
+
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+      in.defaultReadObject();
+      initTransients();
+    }
+
+    private void initTransients() {
+      storageRpc = options.storageRpc();
+      retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
+      storageObject = blob.toPb();
+    }
+
+    private void validateOpen() throws IOException {
+      if (!isOpen) {
+        throw new IOException("stream is closed");
+      }
     }
 
     @Override
     public int write(ByteBuffer byteBuffer) throws IOException {
-      int size = Math.min(byteBuffer.remaining(), 1024 * 1024);
+      validateOpen();
+      if (byteBuffer.remaining() < (buffer.length - bufferLimit)) {
+        byteBuffer.get(buffer, bufferLimit, byteBuffer.remaining());
+        // write buffer if full
+      }
+      int toWrite = buffer.length + byteBuffer.remaining() /
+      int size = Math.min(b, 1024 * 1024);
       byte[] bytes = new byte[size];
       byteBuffer.get(bytes);
       // todo: Use retry helper on retriable failures
@@ -457,12 +491,16 @@ public int write(ByteBuffer byteBuffer) throws IOException {
 
     @Override
     public boolean isOpen() {
-      return false;
+      return isOpen;
     }
 
     @Override
     public void close() throws IOException {
-
+      if (isOpen) {
+        storageRpc.write(buffer, storageObject, uploadId, position, true);
+        position += buffer.length;
+        isOpen = false;
+      }
     }
   }
 

From d4d9427abc0ae275c2244f83db05127f657745f7 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 6 May 2015 14:35:48 -0700
Subject: [PATCH 199/732] add jar with dependencies

---
 pom.xml | 79 ++++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 50 insertions(+), 29 deletions(-)

diff --git a/pom.xml b/pom.xml
index 06b070ecf874..74a221fea848 100644
--- a/pom.xml
+++ b/pom.xml
@@ -50,14 +50,14 @@
   
   
     
-        com.google.auth
-        google-auth-library-credentials
-        0.1.0
+      com.google.auth
+      google-auth-library-credentials
+      0.1.0
     
     
-        com.google.auth
-        google-auth-library-oauth2-http
-        0.1.0
+      com.google.auth
+      google-auth-library-oauth2-http
+      0.1.0
     
     
       com.google.http-client
@@ -178,7 +178,7 @@
           
         
         
-        
+          
+            jar-with-dependencies
+          
+        
+        
+          
+            make-assembly
+            
+            package
+            
+              single
+            
+          
+        
+      
     
   
 

From e84ff219c770d8fe6e9f9dde337208cff9fd3027 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 6 May 2015 16:28:51 -0700
Subject: [PATCH 200/732] work in progress

---
 .../com/google/gcloud/ServiceOptions.java     | 15 +++-
 .../datastore/DatastoreServiceOptions.java    |  2 +-
 .../gcloud/examples/StorageExample.java       | 23 +++---
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../gcloud/storage/StorageServiceImpl.java    | 73 ++++++++++++++-----
 .../gcloud/storage/StorageServiceOptions.java |  2 +-
 7 files changed, 83 insertions(+), 36 deletions(-)

diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 0b03e7951e86..391631a695c0 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -36,6 +36,7 @@
 import java.lang.reflect.Method;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Set;
 import java.util.regex.Matcher;
@@ -184,9 +185,13 @@ protected static String googleCloudProjectId() {
     } catch (IOException ignore) {
       // ignore
     }
-    String configDir = System.getenv("CLOUDSDK_CONFIG");
-    if (configDir == null) {
-      configDir = new File(System.getProperty("user.home"), "/.config/gcloud/").getPath();
+   File configDir;
+    if (System.getenv().containsKey("CLOUDSDK_CONFIG")) {
+      configDir = new File(System.getenv("CLOUDSDK_CONFIG"));
+    } else if (isWindows() &&  System.getenv().containsKey("APPDATA")) {
+      configDir = new File(System.getenv("APPDATA"), "gcloud");
+    } else {
+      configDir = new File(System.getProperty("user.home"), ".config/gcloud");
     }
     try (BufferedReader reader =
         new BufferedReader(new FileReader(new File(configDir, "properties")))) {
@@ -216,6 +221,10 @@ protected static String googleCloudProjectId() {
     return null;
   }
 
+  private static boolean isWindows() {
+    return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).indexOf("windows") > -1;
+  }
+
   protected static String getAppEngineProjectId() {
     // TODO(ozarov): An alternative to reflection would be to depend on AE api jar:
     // http://mvnrepository.com/artifact/com.google.appengine/appengine-api-1.0-sdk/1.2.0
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index b1b1ee31638c..9378549db3d2 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -199,7 +199,7 @@ DatastoreRpc datastoreRpc() {
     return ServiceRpcProvider.datastore(this);
   }
 
-  public static DatastoreServiceOptions defaultIntance() {
+  public static DatastoreServiceOptions defaultInstance() {
     return builder().build();
   }
 
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 0f9650a63bcc..aa38527ec8fc 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -32,6 +32,7 @@
 import com.google.gcloud.storage.StorageServiceOptions;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -192,9 +193,16 @@ public String params() {
   private static class UploadAction extends StorageAction> {
     @Override
     public void run(StorageService storage, Tuple tuple) throws Exception {
-      if (Files.size(tuple.x()) > 1_000_000) {
-        // todo: upload via streaming API
-        throw new IllegalArgumentException("file is too big");
+      if (Files.size(tuple.x()) > 1024) {
+        try (BlobWriteChannel writer = storage.writer(tuple.y())) {
+          byte[] buffer = new byte[1024];
+          try (InputStream input = Files.newInputStream(tuple.x())) {
+            int limit;
+            while ((limit = input.read(buffer)) >= 0) {
+              writer.write(ByteBuffer.wrap(buffer, 0, limit));
+            }
+          }
+        }
       } else {
         byte[] bytes = Files.readAllBytes(tuple.x());
         System.out.println(storage.create(tuple.y(), bytes));
@@ -312,15 +320,6 @@ public static void printUsage() {
   @SuppressWarnings("unchecked")
   public static void main(String... args) throws Exception {
     if (args.length < 1) {
-
-      System.out.println("KOKO.start -------------------------------");
-      StorageService storage =
-          StorageServiceFactory.instance().get(StorageServiceOptions.defaultInstnace());
-      BlobWriteChannel writer = storage
-          .writer(Blob.of("ozarov-javamrsample.appspot.com", "bla"));
-      writer.write(ByteBuffer.wrap("hello world".getBytes()));
-      writer.close();
-      System.out.println("KOKO.end -------------------------------");
       System.out.println("Missing required project id and action");
       printUsage();
       return;
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 502979953221..c165cf5504ed 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray(); // todo: optimize and eliminate unnecessary copy
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, StorageObject to, String uploadId, long position, boolean last)
      throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, bytes));
      long limit = position + bytes.length;
      String range = position + "-" + (bytes.length - 1);
      httpRequest.getHeaders().setContentRange("bytes " + range + (last ? "/" + limit : "/*"));
      HttpResponse response = httpRequest.execute();
      int code = response.getStatusCode();
      if (!last && code != 308 || last && (code != 200 || code != 201)) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, int offset, int length, StorageObject to, String uploadId,
      long position, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, bytes, offset, length));
      long limit = position + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(position).append('-').append(limit - 1).append('/');
      if (last) {
        range.append('*');
      } else {
        range.append(limit);
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      HttpResponse response = httpRequest.execute();
      int code = response.getStatusCode();
      if (!last && code != 308 || last && (code != 200 || code != 201)) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index e3e28c2dc56f..be6bb96cb345 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(byte[] bytes, StorageObject object, String uploadId, long position, boolean last)
      throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(byte[] bytes, int offset, int length, StorageObject object, String uploadId,
      long position, boolean last) throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index bfe1d78f8e03..aea7b0c96016 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -18,6 +18,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.gcloud.RetryHelper.runWithRetries;
+import static java.util.concurrent.Executors.callable;
 
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Function;
@@ -428,14 +429,14 @@ public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
   private static class BlobWriterChannelImpl implements BlobWriteChannel {
 
     private static final int CHUNK_SIZE = 256 * 1024;
-    private static final int MIN_BUFFER_SIZE = CHUNK_SIZE * 4;
+    private static final int COMPACT_THRESHOLD = (int) Math.round(CHUNK_SIZE * 0.8);
 
     private final StorageServiceOptions options;
     private final Blob blob;
     private final String uploadId;
     private int position;
-    private byte[] buffer = new byte[MIN_BUFFER_SIZE];
-    private int bufferLimit;
+    private byte[] buffer = new byte[CHUNK_SIZE];
+    private int limit;
     private boolean isOpen;
 
     private transient StorageRpc storageRpc;
@@ -451,12 +452,45 @@ public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
     }
 
     private void writeObject(ObjectOutputStream out) throws IOException {
+      if (!isOpen) {
+        out.defaultWriteObject();
+        return;
+      }
+      flush();
+      byte[] temp = buffer;
+      if (limit < COMPACT_THRESHOLD) {
+        buffer = Arrays.copyOf(buffer, limit);
+      }
       out.defaultWriteObject();
+      buffer = temp;
+    }
+
+    private void flush() {
+      if (limit >= CHUNK_SIZE) {
+        final int length = limit - limit % CHUNK_SIZE;
+        runWithRetries(callable(new Runnable() {
+          @Override
+          public void run() {
+            System.out.println("Going to flush-> " + length + " bytes");
+            storageRpc.write(buffer, 0, length, storageObject, uploadId, position, false);
+          }
+        }));
+        position += length;
+        limit -= length;
+        byte[] temp = new byte[CHUNK_SIZE];
+        System.arraycopy(buffer, length, temp, 0, limit);
+        buffer = temp;
+      }
     }
 
     private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
       in.defaultReadObject();
-      initTransients();
+      if (isOpen) {
+        if (buffer.length < CHUNK_SIZE) {
+          buffer = Arrays.copyOf(buffer, CHUNK_SIZE);
+        }
+        initTransients();
+      }
     }
 
     private void initTransients() {
@@ -473,20 +507,18 @@ private void validateOpen() throws IOException {
 
     @Override
     public int write(ByteBuffer byteBuffer) throws IOException {
+      System.out.println("Going to write-> " + byteBuffer.remaining() + " bytes");
       validateOpen();
-      if (byteBuffer.remaining() < (buffer.length - bufferLimit)) {
-        byteBuffer.get(buffer, bufferLimit, byteBuffer.remaining());
-        // write buffer if full
+      int toWrite = byteBuffer.remaining();
+      if (buffer.length - limit >= toWrite) {
+        byteBuffer.get(buffer, limit, toWrite);
+      } else {
+        buffer = Arrays.copyOf(buffer, buffer.length + toWrite);
+        byteBuffer.get(buffer, limit, toWrite);
       }
-      int toWrite = buffer.length + byteBuffer.remaining() /
-      int size = Math.min(b, 1024 * 1024);
-      byte[] bytes = new byte[size];
-      byteBuffer.get(bytes);
-      // todo: Use retry helper on retriable failures
-      System.out.println("Koko. BlobWriterChannelImpl.write");
-      options.storageRpc().write(bytes, blob.toPb(), uploadId, position, false);
-      position += size;
-      return size;
+      limit += toWrite;
+      flush();
+      return toWrite;
     }
 
     @Override
@@ -497,9 +529,16 @@ public boolean isOpen() {
     @Override
     public void close() throws IOException {
       if (isOpen) {
-        storageRpc.write(buffer, storageObject, uploadId, position, true);
+        runWithRetries(callable(new Runnable() {
+          @Override
+          public void run() {
+            System.out.println("Going to close-> " + limit + " bytes");
+            storageRpc.write(buffer, 0, limit, storageObject, uploadId, position, true);
+          }
+        }));
         position += buffer.length;
         isOpen = false;
+        buffer = null;
       }
     }
   }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 24dea9c91788..23ebcb0915fd 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -121,7 +121,7 @@ private static String defaultProject() {
     return projectId != null ? projectId : googleCloudProjectId();
   }
 
-  public static StorageServiceOptions defaultInstnace() {
+  public static StorageServiceOptions defaultInstance() {
     return builder().build();
   }
 

From 01d873ef0e5d9c1eb285fe9d33259ab2b0e3e9c8 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 6 May 2015 20:01:49 -0700
Subject: [PATCH 201/732] complete output writer

---
 .../com/google/gcloud/examples/StorageExample.java |  8 +++++++-
 .../com/google/gcloud/spi/DefaultStorageRpc.java   |  2 +-
 .../java/com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../google/gcloud/storage/StorageServiceImpl.java  | 14 ++++++--------
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index aa38527ec8fc..68bfbda59482 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -40,6 +40,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Random;
 
 /**
  * An example of using the Google Cloud Storage.
@@ -194,12 +195,17 @@ private static class UploadAction extends StorageAction> {
     @Override
     public void run(StorageService storage, Tuple tuple) throws Exception {
       if (Files.size(tuple.x()) > 1024) {
+        Random rnd = new Random();
         try (BlobWriteChannel writer = storage.writer(tuple.y())) {
           byte[] buffer = new byte[1024];
           try (InputStream input = Files.newInputStream(tuple.x())) {
             int limit;
             while ((limit = input.read(buffer)) >= 0) {
-              writer.write(ByteBuffer.wrap(buffer, 0, limit));
+              try {
+                writer.write(ByteBuffer.wrap(buffer, 0, limit));
+              } catch (Exception ex) {
+                ex.printStackTrace();
+              }
             }
           }
         }
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index c165cf5504ed..ab49d0b8c125 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(byte[] bytes, int offset, int length, StorageObject to, String uploadId,
      long position, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, bytes, offset, length));
      long limit = position + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(position).append('-').append(limit - 1).append('/');
      if (last) {
        range.append('*');
      } else {
        range.append(limit);
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      HttpResponse response = httpRequest.execute();
      int code = response.getStatusCode();
      if (!last && code != 308 || last && (code != 200 || code != 201)) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/src/main/java/com/google/gcloud/spi/StorageRpc.java
index be6bb96cb345..ab1e9affbbce 100644
--- a/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(byte[] bytes, int offset, int length, StorageObject object, String uploadId,
      long position, boolean last) throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException;
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index aea7b0c96016..fa911e431997 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -437,7 +437,7 @@ private static class BlobWriterChannelImpl implements BlobWriteChannel {
     private int position;
     private byte[] buffer = new byte[CHUNK_SIZE];
     private int limit;
-    private boolean isOpen;
+    private boolean isOpen = true;
 
     private transient StorageRpc storageRpc;
     private transient RetryParams retryParams;
@@ -471,8 +471,7 @@ private void flush() {
         runWithRetries(callable(new Runnable() {
           @Override
           public void run() {
-            System.out.println("Going to flush-> " + length + " bytes");
-            storageRpc.write(buffer, 0, length, storageObject, uploadId, position, false);
+            storageRpc.write(uploadId, buffer, 0, storageObject, position, length, false);
           }
         }));
         position += length;
@@ -507,13 +506,13 @@ private void validateOpen() throws IOException {
 
     @Override
     public int write(ByteBuffer byteBuffer) throws IOException {
-      System.out.println("Going to write-> " + byteBuffer.remaining() + " bytes");
       validateOpen();
       int toWrite = byteBuffer.remaining();
-      if (buffer.length - limit >= toWrite) {
+      int spaceInBuffer = buffer.length - limit;
+      if (spaceInBuffer >= toWrite) {
         byteBuffer.get(buffer, limit, toWrite);
       } else {
-        buffer = Arrays.copyOf(buffer, buffer.length + toWrite);
+        buffer = Arrays.copyOf(buffer, buffer.length + toWrite - spaceInBuffer);
         byteBuffer.get(buffer, limit, toWrite);
       }
       limit += toWrite;
@@ -532,8 +531,7 @@ public void close() throws IOException {
         runWithRetries(callable(new Runnable() {
           @Override
           public void run() {
-            System.out.println("Going to close-> " + limit + " bytes");
-            storageRpc.write(buffer, 0, limit, storageObject, uploadId, position, true);
+            storageRpc.write(uploadId, buffer, 0, storageObject, position, limit, true);
           }
         }));
         position += buffer.length;

From b68088c647810672af21dc4a0453376c24e4e023 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 6 May 2015 20:27:26 -0700
Subject: [PATCH 202/732] remove leftover

---
 src/main/java/com/google/gcloud/examples/StorageExample.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 68bfbda59482..b0b7a81b4656 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -40,7 +40,6 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Random;
 
 /**
  * An example of using the Google Cloud Storage.
@@ -195,7 +194,6 @@ private static class UploadAction extends StorageAction> {
     @Override
     public void run(StorageService storage, Tuple tuple) throws Exception {
       if (Files.size(tuple.x()) > 1024) {
-        Random rnd = new Random();
         try (BlobWriteChannel writer = storage.writer(tuple.y())) {
           byte[] buffer = new byte[1024];
           try (InputStream input = Files.newInputStream(tuple.x())) {

From d4368c56ea9599f1ec82c02972d2e8b34587eca5 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 7 May 2015 10:19:04 -0700
Subject: [PATCH 203/732] change get to download

---
 .../gcloud/examples/StorageExample.java       | 57 +++++++++++++++----
 1 file changed, 46 insertions(+), 11 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index b0b7a81b4656..59b3cb249ff8 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,8 +16,6 @@
 
 package com.google.gcloud.examples;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 import com.google.gcloud.spi.StorageRpc.Tuple;
 import com.google.gcloud.storage.BatchRequest;
 import com.google.gcloud.storage.BatchResponse;
@@ -31,9 +29,13 @@
 import com.google.gcloud.storage.StorageServiceFactory;
 import com.google.gcloud.storage.StorageServiceOptions;
 
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintStream;
 import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -52,9 +54,10 @@
  * 
  • compile using maven - {@code mvn compile}
    • 
  * 
  • run using maven -
  * {@code mvn exec:java -Dexec.mainClass="com.google.gcloud.examples.StorageExample"
- * -Dexec.args="[] list []| info [ []]| get  |
- *  upload   []| delete  +|
- *  cp    | compose  + "}
+ * -Dexec.args="[] list []| info [ []]|
+ *  download   [local_file]| upload   []|
+ *  delete  +| cp    |
+ *  compose  + "}
  * 
    • 
  * 
  */
@@ -229,26 +232,58 @@ public String params() {
     }
   }
 
-  private static class GetAction extends BlobAction {
+  private static class DownloadAction extends StorageAction> {
+
     @Override
-    public void run(StorageService storage, Blob blob) throws IOException {
-      blob = storage.get(blob);
+    public void run(StorageService storage, Tuple tuple) throws IOException {
+      Blob blob = storage.get(tuple.x());
       if (blob == null) {
         System.out.println("No such object");
         return;
       }
+      PrintStream writeTo = System.out;
+      if (tuple.y() != null) {
+        writeTo = new PrintStream(new FileOutputStream(tuple.y().toFile()));
+      }
       if (blob.size() < 1024) {
-        System.out.println(new String(storage.load(blob), UTF_8));
+        writeTo.write(storage.load(blob));
       } else {
         try (BlobReadChannel reader = storage.reader(blob)) {
+          WritableByteChannel channel = Channels.newChannel(writeTo);
           ByteBuffer bytes = ByteBuffer.allocate(64 * 1024);
           while (reader.read(bytes) > 0) {
             bytes.flip();
-            System.out.print(UTF_8.decode(bytes));
+            channel.write(bytes);
             bytes.clear();
           }
         }
       }
+      if (tuple.y() == null) {
+        writeTo.println();
+      } else {
+        writeTo.close();
+      }
+    }
+
+    @Override
+    Tuple parse(String... args) {
+      if (args.length < 2 || args.length > 3) {
+        throw new IllegalArgumentException();
+      }
+      Path path = null;
+      if (args.length > 2) {
+        path = Paths.get(args[2]);
+        if (Files.isDirectory(path)) {
+          path = path.resolve(Paths.get(args[1]).getFileName());
+        }
+      }
+      String blob = args.length < 3 ? path.getFileName().toString() : args[2];
+      return Tuple.of(Blob.of(args[0], args[1]), path);
+    }
+
+    @Override
+    public String params() {
+      return "  [local_file]";
     }
   }
 
@@ -302,7 +337,7 @@ public String params() {
     ACTIONS.put("delete", new DeleteAction());
     ACTIONS.put("list", new ListAction());
     ACTIONS.put("upload", new UploadAction());
-    ACTIONS.put("get", new GetAction());
+    ACTIONS.put("download", new DownloadAction());
     ACTIONS.put("cp", new CopyAction());
     ACTIONS.put("compose", new ComposeAction());
   }

From 425ae6f5d6d6eaacbcd51c304505dc829183441a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 7 May 2015 11:31:27 -0700
Subject: [PATCH 204/732] fix build

---
 .../com/google/gcloud/storage/StorageService.java  |  2 ++
 .../gcloud/datastore/DatastoreServiceTest.java     | 14 +++++++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 71026fda6078..f52e4d667d01 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -425,4 +425,6 @@ public static Builder builder() {
    * @throws StorageServiceException upon failure
    */
   BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
+
+
 }
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index c9ff5d1632ab..f79441945489 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -35,6 +35,7 @@
 import com.google.gcloud.datastore.StructuredQuery.PropertyFilter;
 import com.google.gcloud.spi.DatastoreRpc;
 import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
+import com.google.gcloud.spi.DatastoreRpcFactory;
 
 import org.easymock.EasyMock;
 import org.junit.AfterClass;
@@ -637,18 +638,21 @@ public void testRetires() throws Exception {
         DatastoreV1.LookupRequest.newBuilder().addKey(KEY1.toPb()).build();
     DatastoreV1.LookupResponse responsePb = DatastoreV1.LookupResponse.newBuilder()
         .addFound(EntityResult.newBuilder().setEntity(ENTITY1.toPb())).build();
-    DatastoreRpc mock = EasyMock.createStrictMock(DatastoreRpc.class);
-    EasyMock.expect(mock.lookup(requestPb))
+    DatastoreRpcFactory rpcFactoryMock = EasyMock.createStrictMock(DatastoreRpcFactory.class);
+    DatastoreRpc rpcMock = EasyMock.createStrictMock(DatastoreRpc.class);
+    EasyMock.expect(rpcFactoryMock.create(EasyMock.anyObject(DatastoreServiceOptions.class)))
+        .andReturn(rpcMock);
+    EasyMock.expect(rpcMock.lookup(requestPb))
         .andThrow(new DatastoreRpc.DatastoreRpcException(Reason.UNAVAILABLE))
         .andReturn(responsePb);
-    EasyMock.replay(mock);
+    EasyMock.replay(rpcFactoryMock, rpcMock);
     DatastoreServiceOptions options = this.options.toBuilder()
         .retryParams(RetryParams.getDefaultInstance())
-        .datastoreRpc(mock)
+        .serviceRpcFactory(rpcFactoryMock)
         .build();
     DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
     Entity entity = datastore.get(KEY1);
     assertEquals(ENTITY1, entity);
-    EasyMock.verify(mock);
+    EasyMock.verify(rpcFactoryMock, rpcMock);
   }
 }

From 59bc7079bbd74e1a3393e7173ba1f2eea73db74f Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 7 May 2015 17:42:26 -0700
Subject: [PATCH 205/732] replace dataset with projectId

---
 README.md                                     |  2 +-
 .../com/google/gcloud/ServiceOptions.java     | 30 ++++++++++++++-
 .../com/google/gcloud/datastore/BaseKey.java  | 38 +++++++++----------
 .../gcloud/datastore/DatastoreHelper.java     |  2 +-
 .../gcloud/datastore/DatastoreService.java    |  4 +-
 .../datastore/DatastoreServiceOptions.java    | 35 +++++------------
 .../gcloud/datastore/IncompleteKey.java       | 24 ++++++------
 .../java/com/google/gcloud/datastore/Key.java | 26 ++++++-------
 .../google/gcloud/datastore/KeyFactory.java   | 22 +++++------
 .../gcloud/datastore/QueryResultsImpl.java    |  2 +-
 .../google/gcloud/datastore/Validator.java    | 12 +++---
 .../google/gcloud/datastore/package-info.java |  3 +-
 .../gcloud/examples/DatastoreExample.java     | 10 ++---
 .../gcloud/examples/StorageExample.java       |  2 +-
 .../gcloud/spi/DefaultDatastoreRpc.java       |  2 +-
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../gcloud/storage/StorageServiceOptions.java | 28 +-------------
 .../google/gcloud/datastore/BaseKeyTest.java  | 18 ++++-----
 .../gcloud/datastore/DatastoreHelperTest.java |  4 +-
 .../DatastoreServiceOptionsTest.java          | 10 ++---
 .../datastore/DatastoreServiceTest.java       | 20 +++++-----
 .../gcloud/datastore/IncompleteKeyTest.java   |  6 +--
 .../gcloud/datastore/KeyFactoryTest.java      | 24 ++++++------
 .../gcloud/datastore/LocalGcdHelper.java      | 26 ++++++-------
 .../gcloud/datastore/SerializationTest.java   |  2 +-
 .../gcloud/storage/SerializationTest.java     |  4 +-
 26 files changed, 174 insertions(+), 184 deletions(-)

diff --git a/README.md b/README.md
index 2da2f0f859fd..68c9db99c743 100644
--- a/README.md
+++ b/README.md
@@ -64,7 +64,7 @@ import com.google.gcloud.datastore.Entity;
 import com.google.gcloud.datastore.Key;
 import com.google.gcloud.datastore.KeyFactory;
 
-DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
+DatastoreServiceOptions options = DatastoreServiceOptions.builder().projectId(PROJECT_ID).build();
 DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
 KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
 Key key = keyFactory.newKey(keyName);
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index 391631a695c0..a72cb74185c7 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -18,6 +18,7 @@
 
 
 import static com.google.common.base.MoreObjects.firstNonNull;
+import static com.google.common.base.Preconditions.checkNotNull;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
@@ -46,7 +47,9 @@ public abstract class ServiceOptions> implemen
 
   private static final String DEFAULT_HOST = "https://www.googleapis.com";
   private static final long serialVersionUID = 1203687993961393350L;
+  private static final String PROJECT_ENV_NAME = "default_project_id";
 
+  private final String projectId;
   private final String host;
   private final HttpTransportFactory httpTransportFactory;
   private final AuthCredentials authCredentials;
@@ -81,9 +84,12 @@ public HttpTransport create() {
     }
   }
 
+
+
   protected abstract static class Builder,
       B extends Builder> {
 
+    private String projectId;
     private String host;
     private HttpTransportFactory httpTransportFactory;
     private AuthCredentials authCredentials;
@@ -93,6 +99,7 @@ protected abstract static class Builder,
     protected Builder() {}
 
     protected Builder(ServiceOptions options) {
+      projectId = options.projectId;
       host = options.host;
       httpTransportFactory = options.httpTransportFactory;
       authCredentials = options.authCredentials;
@@ -107,6 +114,11 @@ protected B self() {
       return (B) this;
     }
 
+    public B projectId(String projectId) {
+      this.projectId = projectId;
+      return self();
+    }
+
     public B host(String host) {
       this.host = host;
       return self();
@@ -134,6 +146,7 @@ public B serviceRpcFactory(ServiceRpcFactory serviceRpcFactory) {
   }
 
   protected ServiceOptions(Builder builder) {
+    projectId = checkNotNull(builder.projectId != null ? builder.projectId : defaultProject());
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransportFactory =
         firstNonNull(builder.httpTransportFactory, DefaultHttpTransportFactory.INSTANCE);
@@ -171,6 +184,14 @@ protected static String appEngineAppId() {
     return System.getProperty("com.google.appengine.application.id");
   }
 
+  protected String defaultProject() {
+    String projectId = System.getProperty(PROJECT_ENV_NAME, System.getenv(PROJECT_ENV_NAME));
+    if (projectId == null) {
+      projectId = getAppEngineProjectId();
+    }
+    return projectId != null ? projectId : googleCloudProjectId();
+  }
+
   protected static String googleCloudProjectId() {
     try {
       URL url = new URL("http://metadata/computeMetadata/v1/project/project-id");
@@ -245,6 +266,10 @@ protected static String getAppEngineProjectId() {
 
   protected abstract Set scopes();
 
+  public String projectId() {
+    return projectId;
+  }
+
   public String host() {
     return host;
   }
@@ -272,12 +297,13 @@ public HttpRequestInitializer httpRequestInitializer() {
 
   @Override
   public int hashCode() {
-    return Objects.hash(host, httpTransportFactory, authCredentials, retryParams,
+    return Objects.hash(projectId, host, httpTransportFactory, authCredentials, retryParams,
         serviceRpcFactory);
   }
 
   protected boolean isEquals(ServiceOptions other) {
-    return Objects.equals(host, other.host)
+    return Objects.equals(projectId, other.projectId)
+        && Objects.equals(host, other.host)
         && Objects.equals(httpTransportFactory, other.httpTransportFactory)
         && Objects.equals(authCredentials, other.authCredentials)
         && Objects.equals(retryParams, other.retryParams)
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/src/main/java/com/google/gcloud/datastore/BaseKey.java
index 3a1ff342e4e3..865b95ed8518 100644
--- a/src/main/java/com/google/gcloud/datastore/BaseKey.java
+++ b/src/main/java/com/google/gcloud/datastore/BaseKey.java
@@ -16,7 +16,7 @@
 
 package com.google.gcloud.datastore;
 
-import static com.google.gcloud.datastore.Validator.validateDataset;
+import static com.google.gcloud.datastore.Validator.validateDatabase;
 import static com.google.gcloud.datastore.Validator.validateKind;
 import static com.google.gcloud.datastore.Validator.validateNamespace;
 
@@ -35,31 +35,31 @@ abstract class BaseKey extends Serializable {
 
   private static final long serialVersionUID = -4671243265877410635L;
 
-  private final transient String dataset;
+  private final transient String projectId;
   private final transient String namespace;
   private final transient ImmutableList path;
 
   abstract static class Builder> {
 
-    String dataset;
+    String projectId;
     String namespace;
     String kind;
     final List ancestors;
 
     private static final int MAX_PATH = 100;
 
-    Builder(String dataset) {
-      this.dataset = validateDataset(dataset);
+    Builder(String projectId) {
+      this.projectId = validateDatabase(projectId);
       ancestors = new LinkedList<>();
     }
 
-    Builder(String dataset, String kind) {
-      this(dataset);
+    Builder(String projectId, String kind) {
+      this(projectId);
       this.kind = validateKind(kind);
     }
 
     Builder(BaseKey copyFrom) {
-      dataset = copyFrom.dataset();
+      projectId = copyFrom.projectId();
       namespace = copyFrom.namespace();
       ancestors = new LinkedList<>(copyFrom.ancestors());
       kind = copyFrom.kind();
@@ -93,8 +93,8 @@ public B kind(String kind) {
       return self();
     }
 
-    public B dataset(String dataset) {
-      this.dataset = validateDataset(dataset);
+    public B projectId(String projectId) {
+      this.projectId = validateDatabase(projectId);
       return self();
     }
 
@@ -106,18 +106,18 @@ public B namespace(String namespace) {
     protected abstract BaseKey build();
   }
 
-  BaseKey(String dataset, String namespace, ImmutableList path) {
+  BaseKey(String projectId, String namespace, ImmutableList path) {
     Preconditions.checkArgument(!path.isEmpty(), "Path must not be empty");
-    this.dataset = dataset;
+    this.projectId = projectId;
     this.namespace = namespace;
     this.path = path;
   }
 
   /**
-   * Returns the key's dataset.
+   * Returns the key's projectId.
    */
-  public String dataset() {
-    return dataset;
+  public String projectId() {
+    return projectId;
   }
 
   /**
@@ -154,7 +154,7 @@ public String kind() {
 
   @Override
   public int hashCode() {
-    return Objects.hash(dataset(), namespace(), path());
+    return Objects.hash(projectId(), namespace(), path());
   }
 
   @Override
@@ -166,7 +166,7 @@ public boolean equals(Object obj) {
       return false;
     }
     BaseKey other = (BaseKey) obj;
-    return Objects.equals(dataset(), other.dataset())
+    return Objects.equals(projectId(), other.projectId())
         && Objects.equals(namespace(), other.namespace())
         && Objects.equals(path(), other.path());
   }
@@ -175,8 +175,8 @@ public boolean equals(Object obj) {
   protected DatastoreV1.Key toPb() {
     DatastoreV1.Key.Builder keyPb = DatastoreV1.Key.newBuilder();
     DatastoreV1.PartitionId.Builder partitionIdPb = DatastoreV1.PartitionId.newBuilder();
-    if (dataset != null) {
-      partitionIdPb.setDatasetId(dataset);
+    if (projectId != null) {
+      partitionIdPb.setDatasetId(projectId);
     }
     if (namespace != null) {
       partitionIdPb.setNamespace(namespace);
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index acc4c1065a38..d07db9ece9b2 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -47,7 +47,7 @@ static Entity add(DatastoreWriter writer, FullEntity entity) {
   }
 
   static KeyFactory newKeyFactory(DatastoreServiceOptions options) {
-    return new KeyFactory(options.dataset(), options.namespace());
+    return new KeyFactory(options.projectId(), options.namespace());
   }
 
   /**
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
index c6401c6a0582..7d27938fd246 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreService.java
@@ -21,7 +21,7 @@
 import java.util.List;
 
 /**
- * An interface for Google Cloud Datastore dataset.
+ * An interface for Google Cloud Datastore.
  */
 public interface DatastoreService extends Service, DatastoreReaderWriter {
 
@@ -66,7 +66,7 @@ interface TransactionCallable {
 
   /**
    * Allocate a unique id for the given key.
-   * The returned key will have the same information (dataset, kind, namespace and ancestors)
+   * The returned key will have the same information (projectId, kind, namespace and ancestors)
    * as the given key and will have a newly assigned id.
    *
    * @throws DatastoreServiceException upon failure
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 9378549db3d2..6bed641c7636 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -16,8 +16,6 @@
 
 package com.google.gcloud.datastore;
 
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static com.google.gcloud.datastore.Validator.validateDataset;
 import static com.google.gcloud.datastore.Validator.validateNamespace;
 
 import com.google.api.services.datastore.DatastoreV1;
@@ -43,7 +41,6 @@ public class DatastoreServiceOptions extends ServiceOptions SCOPES = ImmutableSet.of(DATASTORE_SCOPE, USERINFO_SCOPE);
 
-  private final String dataset;
   private final String namespace;
   private final boolean force;
   private final boolean normalizeDataset;
@@ -51,7 +48,6 @@ public class DatastoreServiceOptions extends ServiceOptions {
 
-    private String dataset;
     private String namespace;
     private boolean force;
     private boolean normalizeDataset = true;
@@ -61,7 +57,6 @@ private Builder() {
 
     private Builder(DatastoreServiceOptions options) {
       super(options);
-      dataset = options.dataset;
       force = options.force;
       namespace = options.namespace;
       normalizeDataset = options.normalizeDataset;
@@ -73,11 +68,6 @@ public DatastoreServiceOptions build() {
       return normalizeDataset ? options.normalize() : options;
     }
 
-    public Builder dataset(String dataset) {
-      this.dataset = validateDataset(dataset);
-      return this;
-    }
-
     public Builder namespace(String namespace) {
       this.namespace = validateNamespace(namespace);
       return this;
@@ -99,7 +89,6 @@ private DatastoreServiceOptions(Builder builder) {
     normalizeDataset = builder.normalizeDataset;
     namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
     force = builder.force;
-    dataset = firstNonNull(builder.dataset, defaultDataset());
   }
 
   private DatastoreServiceOptions normalize() {
@@ -109,7 +98,7 @@ private DatastoreServiceOptions normalize() {
 
     Builder builder = toBuilder();
     builder.normalizeDataset(false);
-    // Replace provided dataset with full dataset (s~xxx, e~xxx,...)
+    // Replace provided project-id with full project-id (s~xxx, e~xxx,...)
     DatastoreV1.LookupRequest.Builder requestPb = DatastoreV1.LookupRequest.newBuilder();
     DatastoreV1.Key key = DatastoreV1.Key.newBuilder()
         .addPathElement(DatastoreV1.Key.PathElement.newBuilder().setKind("__foo__").setName("bar"))
@@ -124,23 +113,20 @@ private DatastoreServiceOptions normalize() {
             Iterables.concat(responsePb.getMissingList(), responsePb.getFoundList()).iterator();
         key = combinedIter.next().getEntity().getKey();
       }
-      builder.dataset(key.getPartitionId().getDatasetId());
+      builder.projectId(key.getPartitionId().getDatasetId());
       return new DatastoreServiceOptions(builder);
     } catch (DatastoreRpcException e) {
       throw DatastoreServiceException.translateAndThrow(e);
     }
   }
 
-  private static String defaultDataset() {
-    String dataset = System.getProperty(DATASET_ENV_NAME, System.getenv(DATASET_ENV_NAME));
-    if (dataset == null) {
-      dataset = appEngineAppId();
+  @Override
+  protected String defaultProject() {
+    String projectId = System.getProperty(DATASET_ENV_NAME, System.getenv(DATASET_ENV_NAME));
+    if (projectId == null) {
+      projectId = appEngineAppId();
     }
-    return dataset != null ? dataset : googleCloudProjectId();
-  }
-
-  public String dataset() {
-    return dataset;
+    return projectId != null ? projectId : super.defaultProject();
   }
 
   public String namespace() {
@@ -177,7 +163,7 @@ public Builder toBuilder() {
 
   @Override
   public int hashCode() {
-    return super.hashCode() ^ Objects.hash(dataset, namespace, force, normalizeDataset);
+    return super.hashCode() ^ Objects.hash(namespace, force, normalizeDataset);
   }
 
   @Override
@@ -186,8 +172,7 @@ public boolean equals(Object obj) {
       return false;
     }
     DatastoreServiceOptions other = (DatastoreServiceOptions) obj;
-    return isEquals(other) && Objects.equals(dataset, other.dataset)
-        && Objects.equals(namespace, other.namespace)
+    return isEquals(other) && Objects.equals(namespace, other.namespace)
         && Objects.equals(force, other.force)
         && Objects.equals(normalizeDataset, other.normalizeDataset);
   }
diff --git a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
index 6a8852f83d75..6134eed2905b 100644
--- a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
+++ b/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
@@ -33,8 +33,8 @@ public class IncompleteKey extends BaseKey {
 
   public static class Builder extends BaseKey.Builder {
 
-    private Builder(String dataset, String kind) {
-      super(dataset, kind);
+    private Builder(String projectId, String kind) {
+      super(projectId, kind);
     }
 
     private Builder(IncompleteKey copyFrom) {
@@ -45,12 +45,12 @@ private Builder(IncompleteKey copyFrom) {
     public IncompleteKey build() {
       ImmutableList path = ImmutableList.builder()
           .addAll(ancestors).add(PathElement.of(kind)).build();
-      return new IncompleteKey(dataset, namespace, path);
+      return new IncompleteKey(projectId, namespace, path);
     }
   }
 
-  IncompleteKey(String dataset, String namespace, ImmutableList path) {
-    super(dataset, namespace, path);
+  IncompleteKey(String projectId, String namespace, ImmutableList path) {
+    super(projectId, namespace, path);
   }
 
   @Override
@@ -59,12 +59,12 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   }
 
   static IncompleteKey fromPb(DatastoreV1.Key keyPb) {
-    String dataset = null;
+    String projectId = null;
     String namespace = null;
     if (keyPb.hasPartitionId()) {
       DatastoreV1.PartitionId partitionIdPb = keyPb.getPartitionId();
       if (partitionIdPb.hasDatasetId()) {
-        dataset = partitionIdPb.getDatasetId();
+        projectId = partitionIdPb.getDatasetId();
       }
       if (partitionIdPb.hasNamespace()) {
         namespace = partitionIdPb.getNamespace();
@@ -79,13 +79,13 @@ static IncompleteKey fromPb(DatastoreV1.Key keyPb) {
     ImmutableList path = pathBuilder.build();
     PathElement leaf = path.get(path.size() - 1);
     if (leaf.nameOrId() != null) {
-      return new Key(dataset, namespace, path);
+      return new Key(projectId, namespace, path);
     }
-    return new IncompleteKey(dataset, namespace, path);
+    return new IncompleteKey(projectId, namespace, path);
   }
 
-  public static Builder builder(String dataset, String kind) {
-    return new Builder(dataset, kind);
+  public static Builder builder(String projectId, String kind) {
+    return new Builder(projectId, kind);
   }
 
   public static Builder builder(IncompleteKey copyFrom) {
@@ -93,6 +93,6 @@ public static Builder builder(IncompleteKey copyFrom) {
   }
 
   public static Builder builder(Key parent, String kind) {
-    return builder(parent.dataset(), kind).namespace(parent.namespace()).ancestors(parent.path());
+    return builder(parent.projectId(), kind).namespace(parent.namespace()).ancestors(parent.path());
   }
 }
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/src/main/java/com/google/gcloud/datastore/Key.java
index e5f7ce6083b0..8b04898ffcd0 100644
--- a/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/src/main/java/com/google/gcloud/datastore/Key.java
@@ -44,13 +44,13 @@ public static final class Builder extends BaseKey.Builder {
     private String name;
     private Long id;
 
-    private Builder(String dataset, String kind, String name) {
-      super(dataset, kind);
+    private Builder(String projectId, String kind, String name) {
+      super(projectId, kind);
       this.name = name;
     }
 
-    private Builder(String dataset, String kind, long id) {
-      super(dataset, kind);
+    private Builder(String projectId, String kind, long id) {
+      super(projectId, kind);
       this.id = id;
     }
 
@@ -94,12 +94,12 @@ public Key build() {
       } else {
         pathBuilder.add(PathElement.of(kind, id));
       }
-      return new Key(dataset, namespace, pathBuilder.build());
+      return new Key(projectId, namespace, pathBuilder.build());
     }
   }
 
-  Key(String dataset, String namespace, ImmutableList path) {
-    super(dataset, namespace, path);
+  Key(String projectId, String namespace, ImmutableList path) {
+    super(projectId, namespace, path);
     Preconditions.checkArgument(nameOrId() != null);
   }
 
@@ -173,12 +173,12 @@ static Key fromPb(DatastoreV1.Key keyPb) {
     return (Key) key;
   }
 
-  public static Builder builder(String dataset, String kind, String name) {
-    return new Builder(dataset, kind, name);
+  public static Builder builder(String projectId, String kind, String name) {
+    return new Builder(projectId, kind, name);
   }
 
-  public static Builder builder(String dataset, String kind, long id) {
-    return new Builder(dataset, kind, id);
+  public static Builder builder(String projectId, String kind, long id) {
+    return new Builder(projectId, kind, id);
   }
 
   public static Builder builder(Key copyFrom) {
@@ -194,13 +194,13 @@ public static Builder builder(IncompleteKey copyFrom, long id) {
   }
 
   public static Builder builder(Key parent, String kind, String name) {
-    Builder builder = builder(parent.dataset(), kind, name);
+    Builder builder = builder(parent.projectId(), kind, name);
     addParentToBuilder(parent, builder);
     return builder;
   }
 
   public static Builder builder(Key parent, String kind, long id) {
-    Builder builder = builder(parent.dataset(), kind, id);
+    Builder builder = builder(parent.projectId(), kind, id);
     addParentToBuilder(parent, builder);
     return builder;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index 5a68f0b8de08..fc288d46510f 100644
--- a/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -20,40 +20,40 @@
 
 /**
  * An helper for creating keys for a specific {@link DatastoreService},
- * using its associated dataset and namespace.
+ * using its associated projectId and namespace.
  */
 public final class KeyFactory extends BaseKey.Builder {
 
-  private final String ds;
+  private final String pi;
   private final String ns;
 
-  public KeyFactory(String dataset) {
-    this(dataset, null);
+  public KeyFactory(String projectId) {
+    this(projectId, null);
   }
 
-  public KeyFactory(String dataset, String namespace) {
-    super(dataset);
+  public KeyFactory(String projectId, String namespace) {
+    super(projectId);
     namespace(namespace);
-    this.ds = dataset;
+    this.pi = projectId;
     this.ns = namespace;
   }
 
   public IncompleteKey newKey() {
     ImmutableList path = ImmutableList.builder()
         .addAll(ancestors).add(PathElement.of(kind)).build();
-    return new IncompleteKey(dataset, namespace, path);
+    return new IncompleteKey(projectId, namespace, path);
   }
 
   public Key newKey(String name) {
     ImmutableList path = ImmutableList.builder()
         .addAll(ancestors).add(PathElement.of(kind, name)).build();
-    return new Key(dataset, namespace, path);
+    return new Key(projectId, namespace, path);
   }
 
   public Key newKey(long id) {
     ImmutableList path = ImmutableList.builder()
         .addAll(ancestors).add(PathElement.of(kind, id)).build();
-    return new Key(dataset, namespace, path);
+    return new Key(projectId, namespace, path);
   }
 
   /**
@@ -61,7 +61,7 @@ public Key newKey(long id) {
    * @return {@code this} for chaining.
    */
   public KeyFactory reset() {
-    dataset(ds);
+    projectId(pi);
     namespace(ns);
     kind = null;
     ancestors.clear();
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java b/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
index fa88722bc8b6..f97c9483e002 100644
--- a/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
+++ b/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
@@ -46,7 +46,7 @@ class QueryResultsImpl extends AbstractIterator implements QueryResults
     this.query = query;
     queryResultType = query.type();
     DatastoreV1.PartitionId.Builder pbBuilder = DatastoreV1.PartitionId.newBuilder();
-    pbBuilder.setDatasetId(datastore.options().dataset());
+    pbBuilder.setDatasetId(datastore.options().projectId());
     if (query.namespace() != null) {
       pbBuilder.setNamespace(query.namespace());
     } else if (datastore.options().namespace() != null) {
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/src/main/java/com/google/gcloud/datastore/Validator.java
index 055a8ff4ebdc..09a3fa7defcc 100644
--- a/src/main/java/com/google/gcloud/datastore/Validator.java
+++ b/src/main/java/com/google/gcloud/datastore/Validator.java
@@ -27,7 +27,7 @@
  */
 final class Validator {
 
-  private static final Pattern DATASET_PATTERN = Pattern.compile(
+  private static final Pattern PROJECT_ID_PATTERN = Pattern.compile(
       "([a-z\\d\\-]{1,100}~)?([a-z\\d][a-z\\d\\-\\.]{0,99}:)?([a-z\\d][a-z\\d\\-]{0,99})");
   private static final int MAX_NAMESPACE_LENGTH = 100;
   private static final Pattern NAMESPACE_PATTERN =
@@ -37,11 +37,11 @@ private Validator() {
     // utility class
   }
 
-  static String validateDataset(String dataset) {
-    checkArgument(!Strings.isNullOrEmpty(dataset), "dataset can't be empty or null");
-    checkArgument(DATASET_PATTERN.matcher(dataset).matches(),
-        "dataset must match the following pattern: " + DATASET_PATTERN.pattern());
-    return dataset;
+  static String validateDatabase(String projectId) {
+    checkArgument(!Strings.isNullOrEmpty(projectId), "projectId can't be empty or null");
+    checkArgument(PROJECT_ID_PATTERN.matcher(projectId).matches(),
+        "projectId must match the following pattern: " + PROJECT_ID_PATTERN.pattern());
+    return projectId;
   }
 
   static String validateNamespace(String namespace) {
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/src/main/java/com/google/gcloud/datastore/package-info.java
index 5c4f6d945bcc..cf32c5bfa2c2 100644
--- a/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -19,7 +19,8 @@
  *
  * 
    A simple usage example:
  * 
     {@code
- * DatastoreServiceOptions options = DatastoreServiceOptions.builder().dataset(DATASET).build();
+ * DatastoreServiceOptions options =
+ *     DatastoreServiceOptions.builder().projectId(PROJECT_ID).build();
  * DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
  * KeyFactory keyFactory = datastore.newKeyFactory().kind(kind);
  * Key key = keyFactory.newKey(keyName);
diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index d2e0aa06d9ae..e153bb5da85a 100644
--- a/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -46,7 +46,7 @@
  * 
  • compile using maven - {@code mvn compile}
    • 
  * 
  • run using maven - {@code mvn exec:java
  * -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample"
- * -Dexec.args="dataset [user] [delete|display|add comment]"}
    • 
+ * -Dexec.args="projectId [user] [delete|display|add comment]"}
  * 
  */
 public class DatastoreExample {
@@ -175,15 +175,15 @@ public static void main(String... args) {
     DatastoreService datastore = null;
     Key key = null;
     if (args.length > 0) {
-      String dataset = args[0];
+      String projectId = args[0];
       // If you want to access a local Datastore running via the gcd sdk, do
 //          DatastoreServiceOptions options = DatastoreServiceOptions.builder()
-//              .dataset(dataset)
+//              .projectId(projectId)
 //              .namespace(NAMESPACE)
 //              .host("http://localhost:8080")
 //              .build();
       DatastoreServiceOptions options = DatastoreServiceOptions.builder()
-          .dataset(dataset)
+          .projectId(projectId)
           .namespace(NAMESPACE)
           .build();
       String name = args.length > 1 ? args[1] : System.getProperty("user.name");
@@ -204,7 +204,7 @@ public static void main(String... args) {
         actionAndParams.append('|');
       }
       actionAndParams.setLength(actionAndParams.length() - 1);
-      System.out.printf("Usage: %s dataset [user] [%s]%n",
+      System.out.printf("Usage: %s projectId [user] [%s]%n",
           DatastoreExample.class.getSimpleName(), actionAndParams);
       return;
     }
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 59b3cb249ff8..89b5c971e915 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -366,7 +366,7 @@ public static void main(String... args) throws Exception {
     StorageServiceOptions.Builder optionsBuilder = StorageServiceOptions.builder();
     StorageAction action;
     if (args.length >= 2 && !ACTIONS.containsKey(args[0])) {
-      optionsBuilder.project(args[0]);
+      optionsBuilder.projectId(args[0]);
       action = ACTIONS.get(args[1]);
       args = Arrays.copyOfRange(args, 2, args.length);
     } else {
diff --git a/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
index df7e309c5903..20bd911b3782 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.dataset())
            .host(options.host())
            .initializer(options.httpRequestInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.projectId())
            .host(options.host())
            .initializer(options.httpRequestInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index ab49d0b8c125..ce107815c435 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.project(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.project())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 23ebcb0915fd..13b14a1fbf76 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -17,7 +17,6 @@
 package com.google.gcloud.storage;
 
 import com.google.common.base.MoreObjects;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import com.google.gcloud.ServiceOptions;
 import com.google.gcloud.spi.ServiceRpcProvider;
@@ -32,15 +31,12 @@ public class StorageServiceOptions extends ServiceOptions SCOPES = ImmutableSet.of(GCS_SCOPE);
   private static final String DEFAULT_PATH_DELIMITER = "/";
-  private static final String PROJECT_ENV_NAME = "default_project_id";
 
-  private final String project;
   private final String pathDelimiter;
 
   public static class Builder extends
       ServiceOptions.Builder {
 
-    private String project;
     private String pathDelimiter;
 
     private Builder() {}
@@ -49,11 +45,6 @@ private Builder(StorageServiceOptions options) {
       super(options);
     }
 
-    public Builder project(String project) {
-      this.project = project;
-      return this;
-    }
-
     public Builder pathDelimiter(String pathDelimiter) {
       this.pathDelimiter = pathDelimiter;
       return this;
@@ -68,8 +59,6 @@ public StorageServiceOptions build() {
   private StorageServiceOptions(Builder builder) {
     super(builder);
     pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
-    project = builder.project != null ? builder.project : defaultProject();
-    Preconditions.checkArgument(project != null, "Missing required project id");
     // todo: consider providing read-timeout
   }
 
@@ -85,10 +74,6 @@ StorageRpc storageRpc() {
     return ServiceRpcProvider.storage(this);
   }
 
-  public String project() {
-    return project;
-  }
-
   public String pathDelimiter() {
     return pathDelimiter;
   }
@@ -100,7 +85,7 @@ public Builder toBuilder() {
 
   @Override
   public int hashCode() {
-    return super.hashCode() ^ Objects.hash(project, pathDelimiter);
+    return super.hashCode() ^ Objects.hash(pathDelimiter);
   }
 
   @Override
@@ -109,16 +94,7 @@ public boolean equals(Object obj) {
       return false;
     }
     StorageServiceOptions other = (StorageServiceOptions) obj;
-    return isEquals(other) && Objects.equals(project, other.project)
-        && Objects.equals(pathDelimiter, other.pathDelimiter);
-  }
-
-  private static String defaultProject() {
-    String projectId = System.getProperty(PROJECT_ENV_NAME, System.getenv(PROJECT_ENV_NAME));
-    if (projectId == null) {
-      projectId = getAppEngineProjectId();
-    }
-    return projectId != null ? projectId : googleCloudProjectId();
+    return isEquals(other) && Objects.equals(pathDelimiter, other.pathDelimiter);
   }
 
   public static StorageServiceOptions defaultInstance() {
diff --git a/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java b/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
index 850a71a127bd..e99e7a60fd0b 100644
--- a/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
@@ -32,12 +32,12 @@ public class BaseKeyTest {
 
   private class Builder extends BaseKey.Builder {
 
-    Builder(String dataset) {
-      super(dataset);
+    Builder(String projectId) {
+      super(projectId);
     }
 
-    Builder(String dataset, String kind) {
-      super(dataset, kind);
+    Builder(String projectId, String kind) {
+      super(projectId, kind);
     }
 
     @Override
@@ -45,7 +45,7 @@ protected BaseKey build() {
       ImmutableList.Builder path = ImmutableList.builder();
       path.addAll(ancestors);
       path.add(PathElement.of(kind));
-      return new BaseKey(dataset, namespace, path.build()) {
+      return new BaseKey(projectId, namespace, path.build()) {
         @Override
         protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
           return null;
@@ -58,9 +58,9 @@ protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
   public void testDataset() throws Exception {
     Builder builder = new Builder("ds1", "k");
     BaseKey key = builder.build();
-    assertEquals("ds1", key.dataset());
-    key = builder.dataset("ds2").build();
-    assertEquals("ds2", key.dataset());
+    assertEquals("ds1", key.projectId());
+    key = builder.projectId("ds2").build();
+    assertEquals("ds2", key.projectId());
   }
 
   @Test(expected = IllegalArgumentException.class)
@@ -71,7 +71,7 @@ public void testBadDatasetInConstructor() throws Exception {
   @Test(expected = IllegalArgumentException.class)
   public void testBadDatasetInSetter() throws Exception {
     Builder builder = new Builder("d", "k");
-    builder.dataset(" ");
+    builder.projectId(" ");
   }
 
   @Test
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index cfe6380ce862..25c0e8b2e691 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -34,12 +34,12 @@ public class DatastoreHelperTest {
   @Test
   public void testNewKeyFactory() {
     DatastoreServiceOptions options = createMock(DatastoreServiceOptions.class);
-    expect(options.dataset()).andReturn("ds1").once();
+    expect(options.projectId()).andReturn("ds1").once();
     expect(options.namespace()).andReturn("ns1").once();
     replay(options);
     KeyFactory keyFactory = DatastoreHelper.newKeyFactory(options);
     Key key = keyFactory.kind("k").newKey("bla");
-    assertEquals("ds1", key.dataset());
+    assertEquals("ds1", key.projectId());
     assertEquals("ns1", key.namespace());
     assertEquals("k", key.kind());
     assertEquals("bla", key.name());
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
index 7274f1ca9e91..a533f4ddc15e 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -33,7 +33,7 @@
 
 public class DatastoreServiceOptionsTest {
 
-  private static final String DATASET = "dataset";
+  private static final String PROJECT_ID = "project_id";
   private DatastoreRpcFactory datastoreRpcFactory;
   private DatastoreRpc datastoreRpc;
   private DatastoreServiceOptions.Builder options;
@@ -45,7 +45,7 @@ public void setUp() throws IOException, InterruptedException {
     options = DatastoreServiceOptions.builder()
         .normalizeDataset(false)
         .serviceRpcFactory(datastoreRpcFactory)
-        .dataset(DATASET)
+        .projectId(PROJECT_ID)
         .host("http://localhost:" + LocalGcdHelper.PORT);
     EasyMock.expect(datastoreRpcFactory.create(EasyMock.anyObject(DatastoreServiceOptions.class)))
         .andReturn(datastoreRpc)
@@ -54,8 +54,8 @@ public void setUp() throws IOException, InterruptedException {
   }
 
   @Test
-  public void testDataset() throws Exception {
-    assertEquals(DATASET, options.build().dataset());
+  public void testProjectId() throws Exception {
+    assertEquals(PROJECT_ID, options.build().projectId());
   }
 
   @Test
@@ -85,7 +85,7 @@ public void testDatastore() throws Exception {
   public void testToBuilder() throws Exception {
     DatastoreServiceOptions original = options.namespace("ns1").force(true).build();
     DatastoreServiceOptions copy = original.toBuilder().build();
-    assertEquals(original.dataset(), copy.dataset());
+    assertEquals(original.projectId(), copy.projectId());
     assertEquals(original.namespace(), copy.namespace());
     assertEquals(original.host(), copy.host());
     assertEquals(original.force(), copy.force());
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index f79441945489..0423cfe2d494 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -53,15 +53,17 @@
 @RunWith(JUnit4.class)
 public class DatastoreServiceTest {
 
-  private static final String DATASET = LocalGcdHelper.DEFAULT_DATASET;
+  private static final String PROJECT_ID = LocalGcdHelper.DEFAULT_PROJECT_ID;
   private static final String KIND1 = "kind1";
   private static final String KIND2 = "kind2";
   private static final String KIND3 = "kind3";
   private static final NullValue NULL_VALUE = NullValue.of();
   private static final StringValue STR_VALUE = StringValue.of("str");
   private static final BooleanValue BOOL_VALUE = BooleanValue.builder(false).indexed(false).build();
-  private static final IncompleteKey INCOMPLETE_KEY1 = IncompleteKey.builder(DATASET, KIND1).build();
-  private static final IncompleteKey INCOMPLETE_KEY2 = IncompleteKey.builder(DATASET, KIND2).build();
+  private static final IncompleteKey INCOMPLETE_KEY1 =
+      IncompleteKey.builder(PROJECT_ID, KIND1).build();
+  private static final IncompleteKey INCOMPLETE_KEY2 =
+      IncompleteKey.builder(PROJECT_ID, KIND2).build();
   private static final Key KEY1 = Key.builder(INCOMPLETE_KEY1, "name").build();
   private static final Key KEY2 = Key.builder(KEY1, KIND2, 1).build();
   private static final Key KEY3 = Key.builder(KEY2).name("bla").build();
@@ -81,7 +83,7 @@ public class DatastoreServiceTest {
       FullEntity.builder(PARTIAL_ENTITY1).remove("str").set("bool", true).
           set("list", LIST_VALUE1.get()).build();
   private static final FullEntity PARTIAL_ENTITY3 =
-      FullEntity.builder(PARTIAL_ENTITY1).key(IncompleteKey.builder(DATASET, KIND3).build())
+      FullEntity.builder(PARTIAL_ENTITY1).key(IncompleteKey.builder(PROJECT_ID, KIND3).build())
           .build();
   private static final Entity ENTITY1 = Entity.builder(KEY1)
       .set("str", STR_VALUE)
@@ -102,15 +104,15 @@ public class DatastoreServiceTest {
 
   @BeforeClass
   public static void beforeClass() throws IOException, InterruptedException {
-    if (!LocalGcdHelper.isActive(DATASET)) {
-      gcdHelper = LocalGcdHelper.start(DATASET);
+    if (!LocalGcdHelper.isActive(PROJECT_ID)) {
+      gcdHelper = LocalGcdHelper.start(PROJECT_ID);
     }
   }
 
   @Before
   public void setUp() throws IOException, InterruptedException {
     options = DatastoreServiceOptions.builder()
-        .dataset(DATASET)
+        .projectId(PROJECT_ID)
         .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
     datastore = DatastoreServiceFactory.instance().get(options);
@@ -285,7 +287,7 @@ public void testNewBatch() {
     Entity entity6 = entities.get(1);
     assertSame(entity4, entities.get(0));
     assertEquals(PARTIAL_ENTITY2.properties(), entity6.properties());
-    assertEquals(PARTIAL_ENTITY2.key().dataset(), entity6.key().dataset());
+    assertEquals(PARTIAL_ENTITY2.key().projectId(), entity6.key().projectId());
     assertEquals(PARTIAL_ENTITY2.key().namespace(), entity6.key().namespace());
     assertEquals(PARTIAL_ENTITY2.key().ancestors(), entity6.key().ancestors());
     assertEquals(PARTIAL_ENTITY2.key().kind(), entity6.key().kind());
@@ -453,7 +455,7 @@ public void testAllocateId() {
     KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND1);
     IncompleteKey pk1 = keyFactory.newKey();
     Key key1 = datastore.allocateId(pk1);
-    assertEquals(key1.dataset(), pk1.dataset());
+    assertEquals(key1.projectId(), pk1.projectId());
     assertEquals(key1.namespace(), pk1.namespace());
     assertEquals(key1.ancestors(), pk1.ancestors());
     assertEquals(key1.kind(), pk1.kind());
diff --git a/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java b/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
index 2650ae9fe183..7edbf133d330 100644
--- a/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
+++ b/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
@@ -26,19 +26,19 @@ public class IncompleteKeyTest {
   @Test
   public void testBuilders() throws Exception {
     IncompleteKey pk1 = IncompleteKey.builder("ds", "kind1").build();
-    assertEquals("ds", pk1.dataset());
+    assertEquals("ds", pk1.projectId());
     assertEquals("kind1", pk1.kind());
     assertTrue(pk1.ancestors().isEmpty());
 
     Key parent = Key.builder("ds", "kind2", 10).build();
     IncompleteKey pk2 = IncompleteKey.builder(parent, "kind3").build();
-    assertEquals("ds", pk2.dataset());
+    assertEquals("ds", pk2.projectId());
     assertEquals("kind3", pk2.kind());
     assertEquals(parent.path(), pk2.ancestors());
 
     assertEquals(pk2, IncompleteKey.builder(pk2).build());
     IncompleteKey pk3 = IncompleteKey.builder(pk2).kind("kind4").build();
-    assertEquals("ds", pk3.dataset());
+    assertEquals("ds", pk3.projectId());
     assertEquals("kind4", pk3.kind());
     assertEquals(parent.path(), pk3.ancestors());
   }
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index aef241cd90bf..ca9ea08294f4 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -27,21 +27,21 @@
 
 public class KeyFactoryTest {
 
-  private static final String DATASET = "dataset";
+  private static final String PROJECT_ID = "projectid";
 
   private KeyFactory keyFactory;
 
   @Before
   public void setUp() {
-    keyFactory = new KeyFactory(DATASET).kind("k");
+    keyFactory = new KeyFactory(PROJECT_ID).kind("k");
   }
 
   @Test
   public void testReset() {
     IncompleteKey key =
-        keyFactory.dataset("ds1").namespace("ns1").ancestors(PathElement.of("p", 1)).build();
+        keyFactory.projectId("ds1").namespace("ns1").ancestors(PathElement.of("p", 1)).build();
     assertEquals("k", key.kind());
-    assertEquals("ds1", key.dataset());
+    assertEquals("ds1", key.projectId());
     assertEquals("ns1", key.namespace());
     assertEquals(1, key.ancestors().size());
 
@@ -54,20 +54,20 @@ public void testReset() {
     keyFactory.kind("k1");
     key = keyFactory.newKey();
     assertEquals("k1", key.kind());
-    assertEquals(DATASET, key.dataset());
+    assertEquals(PROJECT_ID, key.projectId());
     assertNull(key.namespace());
     assertTrue(key.ancestors().isEmpty());
 
-    keyFactory = new KeyFactory(DATASET, "ns1").kind("k");
+    keyFactory = new KeyFactory(PROJECT_ID, "ns1").kind("k");
     key = keyFactory.newKey();
-    assertEquals(DATASET, key.dataset());
+    assertEquals(PROJECT_ID, key.projectId());
     assertEquals("ns1", key.namespace());
-    key = keyFactory.dataset("bla1").namespace("bla2").build();
-    assertEquals("bla1", key.dataset());
+    key = keyFactory.projectId("bla1").namespace("bla2").build();
+    assertEquals("bla1", key.projectId());
     assertEquals("bla2", key.namespace());
     keyFactory.reset().kind("kind");
     key = keyFactory.newKey();
-    assertEquals(DATASET, key.dataset());
+    assertEquals(PROJECT_ID, key.projectId());
     assertEquals("ns1", key.namespace());
     assertEquals("kind", key.kind());
   }
@@ -96,7 +96,7 @@ public void testNewIncompleteKey() throws Exception {
 
   @Test(expected = NullPointerException.class)
   public void testNewIncompleteWithNoKind() {
-    new KeyFactory(DATASET).build();
+    new KeyFactory(PROJECT_ID).build();
   }
 
   private void verifyKey(Key key, String name, String namespace, PathElement... ancestors) {
@@ -111,7 +111,7 @@ private void verifyKey(Key key, Long id, String namespace, PathElement... ancest
 
   private void verifyIncompleteKey(IncompleteKey key, String namespace, PathElement... ancestors) {
     assertEquals("k", key.kind());
-    assertEquals(DATASET, key.dataset());
+    assertEquals(PROJECT_ID, key.projectId());
     assertEquals(namespace, key.namespace());
     assertEquals(ancestors.length, key.ancestors().size());
     Iterator iter = key.ancestors().iterator();
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 521089f78d08..28194a1c9ff4 100644
--- a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -46,11 +46,11 @@
  */
 public class LocalGcdHelper {
 
-  private final String dataset;
+  private final String projectId;
   private Path gcdPath;
   private ProcessStreamReader processReader;
 
-  public static final String DEFAULT_DATASET = "dataset1";
+  public static final String DEFAULT_PROJECT_ID = "projectid1";
   public static final int PORT = 8080;
   private static final String GCD = "gcd-head";
   private static final String GCD_LOC = '/' + GCD + ".zip";
@@ -97,8 +97,8 @@ public static ProcessStreamReader start(Process process, String blockUntil) thro
     }
   }
 
-  public LocalGcdHelper(String dataset) {
-    this.dataset = dataset;
+  public LocalGcdHelper(String projectId) {
+    this.projectId = projectId;
   }
 
   public void start() throws IOException, InterruptedException {
@@ -121,7 +121,7 @@ public void start() throws IOException, InterruptedException {
       }
     }
 
-    File datasetFolder = new File(gcdFolder, GCD + '/' + dataset);
+    File datasetFolder = new File(gcdFolder, GCD + '/' + projectId);
     deleteRecurse(datasetFolder.toPath());
 
     // TODO: if System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
@@ -129,14 +129,14 @@ public void start() throws IOException, InterruptedException {
         .redirectErrorStream(true)
         .directory(new File(gcdFolder, GCD))
         .redirectOutput(new File("/dev/null"))
-        .command("bash", "gcd.sh", "create", "-d", dataset, dataset)
+        .command("bash", "gcd.sh", "create", "-d", projectId, projectId)
         .start();
     temp.waitFor();
 
     temp = new ProcessBuilder()
         .directory(new File(gcdFolder, GCD))
         .redirectErrorStream(true)
-        .command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", dataset)
+        .command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", projectId)
         .start();
     processReader = ProcessStreamReader.start(temp, "Dev App Server is now running");
   }
@@ -200,8 +200,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
     });
   }
 
-  public static LocalGcdHelper start(String dataset) throws IOException, InterruptedException {
-    LocalGcdHelper helper = new LocalGcdHelper(dataset);
+  public static LocalGcdHelper start(String projectId) throws IOException, InterruptedException {
+    LocalGcdHelper helper = new LocalGcdHelper(projectId);
     helper.start();
     return helper;
   }
@@ -210,8 +210,8 @@ public static void main(String... args) throws IOException, InterruptedException
     if (args.length == 1) {
       switch (args[0]) {
         case "START":
-          if (!isActive(DEFAULT_DATASET)) {
-            LocalGcdHelper helper = start(DEFAULT_DATASET);
+          if (!isActive(DEFAULT_PROJECT_ID)) {
+            LocalGcdHelper helper = start(DEFAULT_PROJECT_ID);
             try (FileWriter writer = new FileWriter(".local_gcd_helper")) {
               writer.write(helper.gcdPath.toAbsolutePath().toString());
             }
@@ -235,10 +235,10 @@ public static void main(String... args) throws IOException, InterruptedException
     throw new RuntimeException("expecting only START | STOP");
   }
 
-  public static boolean isActive(String dataset) {
+  public static boolean isActive(String projectId) {
     try {
       StringBuilder urlBuilder = new StringBuilder("http://localhost:").append(PORT);
-      urlBuilder.append("/datastore/v1beta2/datasets/").append(dataset).append("/lookup");
+      urlBuilder.append("/datastore/v1beta2/datasets/").append(projectId).append("/lookup");
       URL url = new URL(urlBuilder.toString());
       try (BufferedReader reader =
                new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) {
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index ed0da1e9ef13..1e827971c7fe 100644
--- a/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -135,7 +135,7 @@ public void testServiceOptions() throws Exception {
     DatastoreServiceOptions options = DatastoreServiceOptions.builder()
         .authCredentials(AuthCredentials.createForAppEngine())
         .normalizeDataset(false)
-        .dataset("ds1")
+        .projectId("ds1")
         .build();
     DatastoreServiceOptions serializedCopy = serializeAndDeserialize(options);
     assertEquals(options, serializedCopy);
diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/src/test/java/com/google/gcloud/storage/SerializationTest.java
index dc340c01480a..7e3cb3e89258 100644
--- a/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -35,14 +35,14 @@ public class SerializationTest {
   @Test
   public void testServiceOptions() throws Exception {
     StorageServiceOptions options = StorageServiceOptions.builder()
-        .project("p1")
+        .projectId("p1")
         .authCredentials(AuthCredentials.createForAppEngine())
         .build();
     StorageServiceOptions serializedCopy = serializeAndDeserialize(options);
     assertEquals(options, serializedCopy);
 
     options = options.toBuilder()
-        .project("p2")
+        .projectId("p2")
         .retryParams(RetryParams.getDefaultInstance())
         .authCredentials(AuthCredentials.noCredentials())
         .pathDelimiter(":")

From 5db8c140326fea76b2dcc2e1e0facefc671deae6 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 7 May 2015 19:36:12 -0700
Subject: [PATCH 206/732] update datastore example

---
 .../gcloud/examples/DatastoreExample.java     | 40 +++++++++----------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index e153bb5da85a..3d7feae66e8c 100644
--- a/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -46,7 +46,7 @@
  * 
  • compile using maven - {@code mvn compile}
    • 
  * 
  • run using maven - {@code mvn exec:java
  * -Dexec.mainClass="com.google.gcloud.examples.DatastoreExample"
- * -Dexec.args="projectId [user] [delete|display|add comment]"}
    • 
+ * -Dexec.args="[projectId] [user] [delete|display|add comment]"}
  * 
  */
 public class DatastoreExample {
@@ -174,25 +174,23 @@ public static void main(String... args) {
     DatastoreAction action = null;
     DatastoreService datastore = null;
     Key key = null;
-    if (args.length > 0) {
-      String projectId = args[0];
-      // If you want to access a local Datastore running via the gcd sdk, do
-//          DatastoreServiceOptions options = DatastoreServiceOptions.builder()
-//              .projectId(projectId)
-//              .namespace(NAMESPACE)
-//              .host("http://localhost:8080")
-//              .build();
-      DatastoreServiceOptions options = DatastoreServiceOptions.builder()
-          .projectId(projectId)
-          .namespace(NAMESPACE)
-          .build();
-      String name = args.length > 1 ? args[1] : System.getProperty("user.name");
-      datastore = DatastoreServiceFactory.instance().get(options);
-      KeyFactory keyFactory = datastore.newKeyFactory().kind(USER_KIND);
-      key = keyFactory.newKey(name);
-      String actionName = args.length > 2 ? args[2].toLowerCase() : DEFAULT_ACTION;
-      action = ACTIONS.get(actionName);
-    }
+    String projectId = args.length > 0 ? args[0] : null;
+    // If you want to access a local Datastore running via the gcd sdk, do
+    //   DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+    //       .projectId(projectId)
+    //       .namespace(NAMESPACE)
+    //       .host("http://localhost:8080")
+    //       .build();
+    DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+        .projectId(projectId)
+        .namespace(NAMESPACE)
+        .build();
+    String name = args.length > 1 ? args[1] : System.getProperty("user.name");
+    datastore = DatastoreServiceFactory.instance().get(options);
+    KeyFactory keyFactory = datastore.newKeyFactory().kind(USER_KIND);
+    key = keyFactory.newKey(name);
+    String actionName = args.length > 2 ? args[2].toLowerCase() : DEFAULT_ACTION;
+    action = ACTIONS.get(actionName);
     if (action == null) {
       StringBuilder actionAndParams = new StringBuilder();
       for (Map.Entry entry : ACTIONS.entrySet()) {
@@ -204,7 +202,7 @@ public static void main(String... args) {
         actionAndParams.append('|');
       }
       actionAndParams.setLength(actionAndParams.length() - 1);
-      System.out.printf("Usage: %s projectId [user] [%s]%n",
+      System.out.printf("Usage: %s [projectId] [user] [%s]%n",
           DatastoreExample.class.getSimpleName(), actionAndParams);
       return;
     }

From a1d29f260ce59ae7dfc9587edd9b364822b95d77 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 7 May 2015 20:29:35 -0700
Subject: [PATCH 207/732] javadoc

---
 .../datastore/DatastoreServiceFactory.java     |  4 +++-
 .../java/com/google/gcloud/storage/Acl.java    |  3 +++
 .../google/gcloud/storage/BatchRequest.java    | 15 ++++++++++++---
 .../java/com/google/gcloud/storage/Blob.java   |  5 +++++
 .../java/com/google/gcloud/storage/Bucket.java |  5 +++++
 .../java/com/google/gcloud/storage/Cors.java   |  3 +++
 .../google/gcloud/storage/StorageService.java  | 18 ++++++++++++++----
 .../gcloud/storage/StorageServiceFactory.java  | 10 +++++++++-
 8 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
index b2cd7c8eec11..ce0887792f98 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
@@ -17,7 +17,9 @@
 package com.google.gcloud.datastore;
 
 
-
+/**
+ * A base class for DatasoreService factories.
+ */
 public abstract class DatastoreServiceFactory {
 
   private static final DatastoreServiceFactory INSTANCE = new DatastoreServiceFactory() {
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 10673b5aeae0..138ba4f3fa3a 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -21,6 +21,9 @@
 
 import java.io.Serializable;
 
+/**
+ *  Access Control List on for buckets or blobs.
+ */
 public final class Acl implements Serializable {
 
   private static final long serialVersionUID = 6435575339887912222L;
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 4f27abd89547..84122a83d4ff 100644
--- a/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -43,14 +43,23 @@ public static class Builder {
 
     private Builder() {}
 
+    /**
+     * Delete the given blob.
+     */
     public void delete(Blob blob, BlobSourceOption... options) {
       toDelete.put(blob, options);
     }
 
+    /**
+     * Update the given blob.
+     */
     public void update(Blob blob, BlobTargetOption... options) {
       toUpdate.put(blob, options);
     }
 
+    /**
+     * Retrieve metadata for the given blob.
+     */
     public void get(Blob blob, BlobSourceOption... options) {
       toGet.put(blob, options);
     }
@@ -66,15 +75,15 @@ private BatchRequest(Builder builder) {
     toGet = ImmutableMap.copyOf(builder.toGet);
   }
 
-  public Map toDelete() {
+  Map toDelete() {
     return toDelete;
   }
 
-  public Map toUpdate() {
+  Map toUpdate() {
     return toUpdate;
   }
 
-  public Map toGet() {
+  Map toGet() {
     return toGet;
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index 6cb8e0eb8ae6..f7053799f719 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -33,6 +33,11 @@
 import java.util.List;
 import java.util.Map;
 
+/**
+ * A Google Storage object.
+ *
+ * @see Concepts and Terminology
+ */
 public class Blob implements Serializable {
 
   private static final long serialVersionUID = 2228487739943277159L;
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 981dc7f788c0..ba1be8522a52 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -40,6 +40,11 @@
 import java.io.Serializable;
 import java.util.List;
 
+/**
+ * A Google Storage bucket.
+ *
+ * @see Concepts and Terminology
+ */
 public final class Bucket implements Serializable {
 
   private static final long serialVersionUID = -3946094202176916586L;
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index ccbe2413042e..366c8d9223bd 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -30,6 +30,9 @@
 import java.net.URISyntaxException;
 import java.util.List;
 
+/**
+ * Cross-Origin Resource Sharing (CORS) configuration for a bucket.
+ */
 public final class Cors implements Serializable {
 
   private static final long serialVersionUID = -8637770919343335655L;
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index f52e4d667d01..a97b20b7253a 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -30,11 +30,13 @@
 import java.util.List;
 import java.util.Set;
 
+/**
+ * An interface for Google Cloud Storage.
+ *
+ * @see Google Cloud Storage
+ */
 public interface StorageService extends Service {
 
-  // todo: provide way for construct signed URLs -
-  // https://cloud.google.com/storage/docs/access-control#Signed-URLs
-
   enum PredefinedAcl {
     AUTHENTICATED_READ("authenticatedRead"),
     ALL_AUTHENTICATED_USERS("allAuthenticatedUsers"),
@@ -347,17 +349,26 @@ public static Builder builder() {
     }
   }
 
+
   /**
+   * Create a new bucket.
+   *
+   * @return a complete bucket information.
    * @throws StorageServiceException upon failure
    */
   Bucket create(Bucket bucket, BucketTargetOption... options);
 
   /**
+   * Create a new blob.
+   *
+   * @return a complete blob information.
    * @throws StorageServiceException upon failure
    */
   Blob create(Blob blob, byte[] content, BlobTargetOption... options);
 
   /**
+   * Returns a complete bucket information.
+   *
    * @throws StorageServiceException upon failure
    */
   Bucket get(Bucket bucket, BucketSourceOption... options);
@@ -426,5 +437,4 @@ public static Builder builder() {
    */
   BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
 
-
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
index a801622082ef..51c7c1812e6f 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
@@ -17,7 +17,9 @@
 package com.google.gcloud.storage;
 
 
-
+/**
+ * A base class for StorageService factories.
+ */
 public abstract class StorageServiceFactory {
 
   private static final StorageServiceFactory INSTANCE = new StorageServiceFactory() {
@@ -27,9 +29,15 @@ public StorageService get(StorageServiceOptions options) {
     }
   };
 
+  /**
+   * Returns the default factory instance.
+   */
   public static StorageServiceFactory instance() {
     return INSTANCE;
   }
 
+  /**
+   * Returns a {@code StorageService} for the given options.
+   */
   public abstract StorageService get(StorageServiceOptions options);
 }

From dbb74fc2d606ff84451a5c77ae668aaad4f805e5 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 8 May 2015 13:03:36 -0700
Subject: [PATCH 208/732] fix example

---
 src/main/java/com/google/gcloud/examples/StorageExample.java | 5 +++--
 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java   | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 89b5c971e915..442e3b84d13b 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -270,14 +270,15 @@ Tuple parse(String... args) {
       if (args.length < 2 || args.length > 3) {
         throw new IllegalArgumentException();
       }
-      Path path = null;
+      Path path;
       if (args.length > 2) {
         path = Paths.get(args[2]);
         if (Files.isDirectory(path)) {
           path = path.resolve(Paths.get(args[1]).getFileName());
         }
+      } else {
+        path = null;
       }
-      String blob = args.length < 3 ? path.getFileName().toString() : args[2];
       return Tuple.of(Blob.of(args[0], args[1]), path);
     }
 
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index ce107815c435..e1c6ba43c0a9 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.EmptyContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
    // TOdo: consider options
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      HttpRequest httpRequest =
          storage.getRequestFactory().buildPostRequest(url, new EmptyContent());
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      JsonFactory jsonFactory = storage.getJsonFactory();
      HttpRequestFactory requestFactory = storage.getRequestFactory();
      HttpRequest httpRequest =
          requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file

From a1c6fa1ac47b717c8d246d3bef878d990646490c Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 8 May 2015 13:29:32 -0700
Subject: [PATCH 209/732] work in progress

---
 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +-
 src/main/java/com/google/gcloud/storage/Blob.java          | 4 ++--
 src/main/java/com/google/gcloud/storage/Bucket.java        | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index e1c6ba43c0a9..7ebbb1340d0f 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object value = option.get(options);
        if (value != null) {
          url.set(option.name(), value.toString());
        }
      }
      JsonFactory jsonFactory = storage.getJsonFactory();
      HttpRequestFactory requestFactory = storage.getRequestFactory();
      HttpRequest httpRequest =
          requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object content = option.get(options);
        if (content != null) {
          url.set(option.value(), content.toString());
        }
      }
      JsonFactory jsonFactory = storage.getJsonFactory();
      HttpRequestFactory requestFactory = storage.getRequestFactory();
      HttpRequest httpRequest =
          requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index f7053799f719..49d763b0f65a 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -153,7 +153,7 @@ public Builder cacheControl(String cacheControl) {
     }
 
     public Builder acl(List acl) {
-      this.acl = ImmutableList.copyOf(acl);
+      this.acl = acl != null ? ImmutableList.copyOf(acl) : null;
       return this;
     }
 
@@ -193,7 +193,7 @@ public Builder mediaLink(String mediaLink) {
     }
 
     public Builder metadata(Map metadata) {
-      this.metadata = ImmutableMap.copyOf(metadata);
+      this.metadata = metadata != null ? ImmutableMap.copyOf(metadata) : null;
       return this;
     }
 
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index ba1be8522a52..f6be9a78b2c6 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -436,17 +436,17 @@ Builder metageneration(Long metageneration) {
     }
 
     public Builder cors(Iterable cors) {
-      this.cors = ImmutableList.copyOf(cors);
+      this.cors = cors != null ? ImmutableList.copyOf(cors) : null;
       return this;
     }
 
     public Builder acl(Iterable acl) {
-      this.acl = ImmutableList.copyOf(acl);
+      this.acl = acl != null ? ImmutableList.copyOf(acl) : null;
       return this;
     }
 
     public Builder defaultAcl(Iterable acl) {
-      this.defaultAcl = ImmutableList.copyOf(acl);
+      this.defaultAcl = acl != null ? ImmutableList.copyOf(acl) : null;
       return this;
     }
 

From 34b6ee9d7d8c603ff190feba31a299f412c6fd37 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 8 May 2015 17:17:07 -0700
Subject: [PATCH 210/732] change sourceOptions

---
 .../gcloud/examples/StorageExample.java       |  18 +-
 .../google/gcloud/storage/BatchRequest.java   |   8 +-
 .../google/gcloud/storage/BatchResponse.java  |   8 +-
 .../com/google/gcloud/storage/Option.java     |   2 +-
 .../google/gcloud/storage/StorageService.java | 154 +++++++++++++-----
 .../gcloud/storage/StorageServiceImpl.java    | 113 +++++++------
 6 files changed, 203 insertions(+), 100 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 442e3b84d13b..b545e8323dcd 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -117,14 +117,14 @@ private static class InfoAction extends BlobsAction {
     public void run(StorageService storage, Blob... blobs) {
       if (blobs.length == 1) {
         if (blobs[0].name().isEmpty()) {
-          System.out.println(storage.get(Bucket.of(blobs[0].bucket())));
+          System.out.println(storage.get(blobs[0].bucket()));
         } else {
-          System.out.println(storage.get(blobs[0]));
+          System.out.println(storage.get(blobs[0].bucket(), blobs[0].name()));
         }
       } else {
         BatchRequest.Builder batch = BatchRequest.builder();
         for (Blob blob : blobs) {
-          batch.get(blob);
+          batch.get(blob.bucket(), blob.name());
         }
         BatchResponse response = storage.apply(batch.build());
         System.out.println(response.gets());
@@ -149,11 +149,11 @@ private static class DeleteAction extends BlobsAction {
     @Override
     public void run(StorageService storage, Blob... blobs) {
       if (blobs.length == 1) {
-        System.out.println(storage.delete(blobs[0]));
+        System.out.println(storage.delete(blobs[0].bucket(), blobs[0].name()));
       } else {
         BatchRequest.Builder batch = BatchRequest.builder();
         for (Blob blob : blobs) {
-          batch.delete(blob);
+          batch.delete(blob.bucket(), blob.name());
         }
         BatchResponse response = storage.apply(batch.build());
         System.out.println(response.deletes());
@@ -236,7 +236,7 @@ private static class DownloadAction extends StorageAction> {
 
     @Override
     public void run(StorageService storage, Tuple tuple) throws IOException {
-      Blob blob = storage.get(tuple.x());
+      Blob blob = storage.get(tuple.x().bucket(), tuple.x().name());
       if (blob == null) {
         System.out.println("No such object");
         return;
@@ -246,9 +246,9 @@ public void run(StorageService storage, Tuple tuple) throws IOExcept
         writeTo = new PrintStream(new FileOutputStream(tuple.y().toFile()));
       }
       if (blob.size() < 1024) {
-        writeTo.write(storage.load(blob));
+        writeTo.write(storage.load(blob.bucket(), blob.name()));
       } else {
-        try (BlobReadChannel reader = storage.reader(blob)) {
+        try (BlobReadChannel reader = storage.reader(blob.bucket(), blob.name())) {
           WritableByteChannel channel = Channels.newChannel(writeTo);
           ByteBuffer bytes = ByteBuffer.allocate(64 * 1024);
           while (reader.read(bytes) > 0) {
@@ -299,7 +299,7 @@ CopyRequest parse(String... args) {
       if (args.length != 4) {
         throw new IllegalArgumentException();
       }
-      return CopyRequest.of(Blob.of(args[0], args[1]), Blob.of(args[2], args[3]));
+      return CopyRequest.of(args[0], args[1], Blob.of(args[2], args[3]));
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 84122a83d4ff..9fbee876cf45 100644
--- a/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -46,8 +46,8 @@ private Builder() {}
     /**
      * Delete the given blob.
      */
-    public void delete(Blob blob, BlobSourceOption... options) {
-      toDelete.put(blob, options);
+    public void delete(String bucket, String blob, BlobSourceOption... options) {
+      toDelete.put(Blob.of(bucket, blob), options);
     }
 
     /**
@@ -60,8 +60,8 @@ public void update(Blob blob, BlobTargetOption... options) {
     /**
      * Retrieve metadata for the given blob.
      */
-    public void get(Blob blob, BlobSourceOption... options) {
-      toGet.put(blob, options);
+    public void get(String bucket, String blob, BlobSourceOption... options) {
+      toGet.put(Blob.of(bucket, blob), options);
     }
 
     public BatchRequest build() {
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index bef428c37a74..0cce1300bcec 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -27,9 +27,11 @@
  */
 public class BatchResponse implements Serializable {
 
-  private List> deleteResult;
-  private List> updateResult;
-  private List> getResult;
+  private static final long serialVersionUID = 1057416839397037706L;
+
+  private final List> deleteResult;
+  private final List> updateResult;
+  private final List> getResult;
 
   public static class Result implements Serializable {
 
diff --git a/src/main/java/com/google/gcloud/storage/Option.java b/src/main/java/com/google/gcloud/storage/Option.java
index 7e130603b79c..798db688c8ec 100644
--- a/src/main/java/com/google/gcloud/storage/Option.java
+++ b/src/main/java/com/google/gcloud/storage/Option.java
@@ -36,7 +36,7 @@ class Option implements Serializable {
 
   Option(StorageRpc.Option rpcOption, Object value) {
     this.rpcOption = checkNotNull(rpcOption);
-    this.value = checkNotNull(value);
+    this.value = value;
   }
 
   StorageRpc.Option rpcOption() {
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index a97b20b7253a..2294704eac33 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.collect.ImmutableList;
@@ -66,6 +67,10 @@ private BucketTargetOption(StorageRpc.Option rpcOption, Object value) {
       super(rpcOption, value);
     }
 
+    private BucketTargetOption(StorageRpc.Option rpcOption) {
+      this(rpcOption, null);
+    }
+
     public static BucketTargetOption predefinedAcl(PredefinedAcl acl) {
       return new BucketTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
@@ -74,8 +79,12 @@ public static BucketTargetOption predefinedDefaultObjectAcl(PredefinedAcl acl) {
       return new BucketTargetOption(StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL, acl.entry());
     }
 
-    public static BucketTargetOption metagenerationMatch(boolean match) {
-      return new BucketTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
+    public static BucketTargetOption metagenerationMatch() {
+      return new BucketTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH);
+    }
+
+    public static BucketTargetOption metagenerationNotMatch() {
+      return new BucketTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH);
     }
   }
 
@@ -83,12 +92,16 @@ class BucketSourceOption extends Option {
 
     private static final long serialVersionUID = 5185657617120212117L;
 
-    private BucketSourceOption(StorageRpc.Option rpcOption, Object value) {
-      super(rpcOption, value);
+    private BucketSourceOption(StorageRpc.Option rpcOption, long metageneration) {
+      super(rpcOption, metageneration);
     }
 
-    public static BucketSourceOption metagenerationMatch(boolean match) {
-      return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
+    public static BucketSourceOption metagenerationMatch(long metageneration) {
+      return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration);
+    }
+
+    public static BucketSourceOption metagenerationNotMatch(long metageneration) {
+      return new BucketSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration);
     }
   }
 
@@ -100,16 +113,28 @@ private BlobTargetOption(StorageRpc.Option rpcOption, Object value) {
       super(rpcOption, value);
     }
 
+    private BlobTargetOption(StorageRpc.Option rpcOption) {
+      this(rpcOption, null);
+    }
+
     public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
       return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
 
-    public static BlobTargetOption generationMath(boolean match) {
-      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, match);
+    public static BlobTargetOption generationMatch() {
+      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH);
     }
 
-    public static BlobTargetOption metagenerationMatch(boolean match) {
-      return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
+    public static BlobTargetOption generationNotMatch() {
+      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH);
+    }
+
+    public static BlobTargetOption metagenerationMatch() {
+      return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_MATCH);
+    }
+
+    public static BlobTargetOption metagenerationNotMatch() {
+      return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH);
     }
   }
 
@@ -117,16 +142,24 @@ class BlobSourceOption extends Option {
 
     private static final long serialVersionUID = -3712768261070182991L;
 
-    private BlobSourceOption(StorageRpc.Option rpcOption, Object value) {
+    private BlobSourceOption(StorageRpc.Option rpcOption, long value) {
       super(rpcOption, value);
     }
 
-    public static BlobSourceOption generationMath(boolean match) {
-      return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH, match);
+    public static BlobSourceOption generationMatch(long generation) {
+      return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_MATCH, generation);
+    }
+
+    public static BlobSourceOption generationNotMatch(long generation) {
+      return new BlobSourceOption(StorageRpc.Option.IF_GENERATION_NOT_MATCH, generation);
     }
 
-    public static BlobSourceOption metagenerationMatch(boolean match) {
-      return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, match);
+    public static BlobSourceOption metagenerationMatch(long metageneration) {
+      return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_MATCH, metageneration);
+    }
+
+    public static BlobSourceOption metagenerationNotMatch(long metageneration) {
+      return new BlobSourceOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metageneration);
     }
   }
 
@@ -226,8 +259,11 @@ public Builder addSource(String... blobs) {
         return addSource(Arrays.asList(blobs));
       }
 
-      public Builder addSource(String blob, long matchGeneration) {
-        sourceBlobs.add(new SourceBlob(blob, matchGeneration));
+      /**
+       * Add a source with a specific generation to match.
+       */
+      public Builder addSource(String blob, long generation) {
+        sourceBlobs.add(new SourceBlob(blob, generation));
         return this;
       }
 
@@ -242,6 +278,7 @@ public Builder targetOptions(BlobTargetOption... options) {
       }
 
       public ComposeRequest build() {
+        checkArgument(!sourceBlobs.isEmpty());
         checkNotNull(target);
         return new ComposeRequest(this);
       }
@@ -278,20 +315,23 @@ class CopyRequest implements Serializable {
 
     private static final long serialVersionUID = -2606508373751748775L;
 
-    private final Blob source;
+    private final String sourceBucket;
+    private final String sourceBlob;
     private final List sourceOptions;
     private final Blob target;
     private final List targetOptions;
 
     public static class Builder {
 
-      private Blob source;
+      private String sourceBucket;
+      private String sourceBlob;
       private final Set sourceOptions = new LinkedHashSet<>();
       private Blob target;
       private final Set targetOptions = new LinkedHashSet<>();
 
-      public Builder source(Blob source) {
-        this.source = source;
+      public Builder source(String bucket, String blob) {
+        this.sourceBucket = bucket;
+        this.sourceBlob = blob;
         return this;
       }
 
@@ -311,21 +351,27 @@ public Builder targetOptions(BlobTargetOption... options) {
       }
 
       public CopyRequest build() {
-        checkNotNull(source);
+        checkNotNull(sourceBucket);
+        checkNotNull(sourceBlob);
         checkNotNull(target);
         return new CopyRequest(this);
       }
     }
 
     private CopyRequest(Builder builder) {
-      source = checkNotNull(builder.source);
+      sourceBucket = checkNotNull(builder.sourceBucket);
+      sourceBlob = checkNotNull(builder.sourceBlob);
       sourceOptions = ImmutableList.copyOf(builder.sourceOptions);
       target = checkNotNull(builder.target);
       targetOptions = ImmutableList.copyOf(builder.targetOptions);
     }
 
-    public Blob source() {
-      return source;
+    public String sourceBucket() {
+      return sourceBucket;
+    }
+
+    public String sourceBlob() {
+      return sourceBlob;
     }
 
     public List sourceOptions() {
@@ -340,8 +386,8 @@ public List targetOptions() {
       return targetOptions;
     }
 
-    public static CopyRequest of(Blob source, Blob target) {
-      return builder().source(source).target(target).build();
+    public static CopyRequest of(String sourceBucket, String sourceBlob, Blob target) {
+      return builder().source(sourceBucket, sourceBlob).target(target).build();
     }
 
     public static Builder builder() {
@@ -349,7 +395,6 @@ public static Builder builder() {
     }
   }
 
-
   /**
    * Create a new bucket.
    *
@@ -367,72 +412,107 @@ public static Builder builder() {
   Blob create(Blob blob, byte[] content, BlobTargetOption... options);
 
   /**
-   * Returns a complete bucket information.
+   * Return the requested bucket.
    *
    * @throws StorageServiceException upon failure
    */
-  Bucket get(Bucket bucket, BucketSourceOption... options);
+  Bucket get(String bucket, BucketSourceOption... options);
 
   /**
+   * Return the requested blob.
+   *
    * @throws StorageServiceException upon failure
    */
-  Blob get(Blob blob, BlobSourceOption... options);
+  Blob get(String bucket, String blob, BlobSourceOption... options);
 
   /**
+   * List the project's buckets.
+   *
    * @throws StorageServiceException upon failure
    */
   ListResult list(BucketListOption... options);
 
   /**
-   * Lists blobs for a bucket.
+   * List the bucket's blobs.
+   *
    * @throws StorageServiceException upon failure
    */
   ListResult list(String bucket, BlobListOption... options);
 
   /**
+   * Update bucket information.
+   *
+   * @return the updated bucket
    * @throws StorageServiceException upon failure
    */
   Bucket update(Bucket bucket, BucketTargetOption... options);
 
   /**
+   * Update blob information.
+   *
+   * @return the updated blob
    * @throws StorageServiceException upon failure
    */
   Blob update(Blob blob, BlobTargetOption... options);
 
   /**
+   * Delete the requested bucket.
+   *
+   * @return true if bucket was deleted
    * @throws StorageServiceException upon failure
    */
-  boolean delete(Bucket bucket, BucketSourceOption... options);
+  boolean delete(String bucket, BucketSourceOption... options);
 
   /**
+   * Delete the requested blob.
+   *
+   * @return true if blob was deleted
    * @throws StorageServiceException upon failure
    */
-  boolean delete(Blob blob, BlobSourceOption... options);
+  boolean delete(String bucket, String blob, BlobSourceOption... options);
 
   /**
+   * Send a compose request.
+   *
+   * @return the composed blob.
    * @throws StorageServiceException upon failure
    */
   Blob compose(ComposeRequest composeRequest);
 
   /**
+   * Send a copy request.
+   *
+   * @return the copied blob.
    * @throws StorageServiceException upon failure
    */
   Blob copy(CopyRequest copyRequest);
 
   /**
+   * Load the content of the given blob.
+   *
+   * @return the blob's content.
    * @throws StorageServiceException upon failure
    */
-  byte[] load(Blob blob, BlobSourceOption... options);
-
+  byte[] load(String bucket, String blob, BlobSourceOption... options);
 
+  /**
+   * Send a batch request.
+   *
+   * @return the batch response
+   * @throws StorageServiceException upon failure
+   */
   BatchResponse apply(BatchRequest batchRequest);
 
   /**
+   * Return a channel for reading the blob's content.
+   *
    * @throws StorageServiceException upon failure
    */
-  BlobReadChannel reader(Blob blob, BlobSourceOption... options);
+  BlobReadChannel reader(String bucket, String blob, BlobSourceOption... options);
 
   /**
+   * Create a blob and return a channel for writing its content.
+   *
    * @throws StorageServiceException upon failure
    */
   BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index fa911e431997..e577cf2c0aa3 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -18,6 +18,15 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.gcloud.RetryHelper.runWithRetries;
+import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
 import static java.util.concurrent.Executors.callable;
 
 import com.google.api.services.storage.model.StorageObject;
@@ -108,9 +117,9 @@ public StorageObject call() {
   }
 
   @Override
-  public Bucket get(Bucket bucket, BucketSourceOption... options) {
-    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(bucket, options);
+  public Bucket get(String bucket, BucketSourceOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = Bucket.of(bucket).toPb();
+    final Map optionsMap = optionMap(options);
     return Bucket.fromPb(runWithRetries(
         new Callable() {
           @Override
@@ -121,9 +130,9 @@ public com.google.api.services.storage.model.Bucket call() {
   }
 
   @Override
-  public Blob get(Blob blob, BlobSourceOption... options) {
-    final StorageObject storedObject = blob.toPb();
-    final Map optionsMap = optionMap(blob, options);
+  public Blob get(String bucket, String blob, BlobSourceOption... options) {
+    final StorageObject storedObject = Blob.of(bucket, blob).toPb();
+    final Map optionsMap = optionMap(options);
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
@@ -134,7 +143,7 @@ public StorageObject call() {
 
   @Override
   public ListResult list(BucketListOption... options) {
-    final Map optionsMap = optionMap(null, null, options);
+    final Map optionsMap = optionMap(options);
     Tuple> result = runWithRetries(
         new Callable>>() {
           @Override
@@ -153,7 +162,7 @@ public Bucket apply(com.google.api.services.storage.model.Bucket bucketPb) {
 
   @Override
   public ListResult list(final String bucket, BlobListOption... options) {
-    final Map optionsMap = optionMap(null, null, options);
+    final Map optionsMap = optionMap(options);
     Tuple> result = runWithRetries(
         new Callable>>() {
           @Override
@@ -196,9 +205,9 @@ public StorageObject call() {
   }
 
   @Override
-  public boolean delete(Bucket bucket, BucketSourceOption... options) {
-    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(bucket, options);
+  public boolean delete(String bucket, BucketSourceOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = Bucket.of(bucket).toPb();
+    final Map optionsMap = optionMap(options);
     return runWithRetries(new Callable() {
       @Override
       public Boolean call() {
@@ -208,9 +217,9 @@ public Boolean call() {
   }
 
   @Override
-  public boolean delete(Blob blob, BlobSourceOption... options) {
-    final StorageObject storageObject = blob.toPb();
-    final Map optionsMap = optionMap(blob, options);
+  public boolean delete(String bucket, String blob, BlobSourceOption... options) {
+    final StorageObject storageObject = Blob.of(bucket, blob).toPb();
+    final Map optionsMap = optionMap(options);
     return runWithRetries(new Callable() {
       @Override
       public Boolean call() {
@@ -240,10 +249,11 @@ public StorageObject call() {
 
   @Override
   public Blob copy(CopyRequest copyRequest) {
-    final StorageObject source = copyRequest.source().toPb();
+    final StorageObject source =
+        Blob.of(copyRequest.sourceBucket(), copyRequest.sourceBlob()).toPb();
     copyRequest.sourceOptions();
-    final Map sourceOptions = optionMap(copyRequest.source().generation(),
-        copyRequest.source().metageneration(), copyRequest.sourceOptions(), true);
+    final Map sourceOptions =
+        optionMap(null, null, copyRequest.sourceOptions(), true);
     final StorageObject target = copyRequest.target().toPb();
     final Map targetOptions = optionMap(copyRequest.target().generation(),
         copyRequest.target().metageneration(), copyRequest.targetOptions());
@@ -256,9 +266,9 @@ public StorageObject call() {
   }
 
   @Override
-  public byte[] load(Blob blob, BlobSourceOption... options) {
-    final StorageObject storageObject = blob.toPb();
-    final Map optionsMap = optionMap(blob, options);
+  public byte[] load(String bucket, String blob, BlobSourceOption... options) {
+    final StorageObject storageObject = Blob.of(bucket, blob).toPb();
+    final Map optionsMap = optionMap(options);
     return runWithRetries(new Callable() {
       @Override
       public byte[] call() {
@@ -421,9 +431,9 @@ public byte[] call() {
   }
 
   @Override
-  public BlobReadChannel reader(Blob blob, BlobSourceOption... options) {
-    Map optionsMap = optionMap(blob, options);
-    return new BlobReadChannelImpl(options(), blob, optionsMap);
+  public BlobReadChannel reader(String bucket, String blob, BlobSourceOption... options) {
+    Map optionsMap = optionMap(options);
+    return new BlobReadChannelImpl(options(), Blob.of(bucket, blob), optionsMap);
   }
 
   private static class BlobWriterChannelImpl implements BlobWriteChannel {
@@ -559,35 +569,46 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
       Object prev = temp.put(option.rpcOption(), option.value());
       checkArgument(prev == null, "Duplicate option %s", option);
     }
-    Boolean value = (Boolean) temp.remove(StorageRpc.Option.DELIMITER);
+    Boolean value = (Boolean) temp.remove(DELIMITER);
     if (Boolean.TRUE.equals(value)) {
-      temp.put(StorageRpc.Option.DELIMITER, options().pathDelimiter());
+      temp.put(DELIMITER, options().pathDelimiter());
     }
-    value = (Boolean) temp.remove(StorageRpc.Option.IF_GENERATION_MATCH);
-    if (value != null) {
-      checkArgument(generation != null, "missing generation value");
-      if (value) {
-        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_GENERATION_MATCH
-            : StorageRpc.Option.IF_GENERATION_MATCH, generation);
-      } else {
-        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH
-            : StorageRpc.Option.IF_GENERATION_NOT_MATCH, generation);
-      }
-    }
-    value = (Boolean) temp.remove(StorageRpc.Option.IF_METAGENERATION_MATCH);
-    if (value != null) {
-      checkArgument(metaGeneration != null, "missing metaGeneration value");
-      if (value) {
-        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH
-            : StorageRpc.Option.IF_METAGENERATION_MATCH, metaGeneration);
-      } else {
-        temp.put(useAsSource ? StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH
-            : StorageRpc.Option.IF_METAGENERATION_NOT_MATCH, metaGeneration);
-      }
+    if (useAsSource) {
+      addToOptionMap(IF_GENERATION_MATCH, IF_SOURCE_GENERATION_MATCH, generation, temp);
+      addToOptionMap(IF_GENERATION_NOT_MATCH, IF_SOURCE_GENERATION_NOT_MATCH, generation, temp);
+      addToOptionMap(IF_METAGENERATION_MATCH, IF_SOURCE_METAGENERATION_MATCH, metaGeneration, temp);
+      addToOptionMap(IF_METAGENERATION_NOT_MATCH,
+          IF_SOURCE_METAGENERATION_NOT_MATCH, metaGeneration, temp);
+    } else {
+      addToOptionMap(IF_GENERATION_MATCH, generation, temp);
+      addToOptionMap(IF_GENERATION_NOT_MATCH, generation, temp);
+      addToOptionMap(IF_METAGENERATION_MATCH, metaGeneration, temp);
+      addToOptionMap(IF_METAGENERATION_NOT_MATCH, metaGeneration, temp);
     }
     return ImmutableMap.copyOf(temp);
   }
 
+  private static  void addToOptionMap(StorageRpc.Option option, T defaultValue,
+      Map map) {
+    addToOptionMap(option, option, defaultValue, map);
+  }
+
+  private static  void addToOptionMap(StorageRpc.Option getOption, StorageRpc.Option putOption,
+      T defaultValue, Map map) {
+    if (map.containsKey(getOption)) {
+      @SuppressWarnings("unchecked")
+      T value = (T) map.remove(getOption);
+      checkArgument(value != null || defaultValue != null,
+          "Option " + getOption.value() + " is missing a value");
+      value = MoreObjects.firstNonNull(value, defaultValue);
+      map.put(putOption, value);
+    }
+  }
+
+  private Map optionMap(Option... options) {
+    return optionMap(null, null, Arrays.asList(options));
+  }
+
   private Map optionMap(Long generation, Long metaGeneration,
       Option... options) {
     return optionMap(generation, metaGeneration, Arrays.asList(options));

From 12cc69645c09d9d68aa765fe136c57f5c914f125 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 8 May 2015 17:41:10 -0700
Subject: [PATCH 211/732] some doc

---
 .../java/com/google/gcloud/storage/package-info.java   | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/src/main/java/com/google/gcloud/storage/package-info.java
index 0aa916666e8c..e938755271eb 100644
--- a/src/main/java/com/google/gcloud/storage/package-info.java
+++ b/src/main/java/com/google/gcloud/storage/package-info.java
@@ -17,7 +17,15 @@
 /**
  * A client to Google Cloud Storage.
  *
- * @see Google Cloud Storageg
+ * 
    A simple usage example:
+ * 
    {@code
+ * StorageServiceOptions options = StorageServiceOptions.builder().projectId("project").build();
+ * StorageService storage = StorageServiceFactory.instance().get(options);
+ * byte[] content = readContent();
+ * Blob blob = storage.create(Blob.of("bucket", "blob_name"), content);
+ * } 
    
+ *
+ * @see Google Cloud Storage
  */
 package com.google.gcloud.storage;
 

From c5ca4503e41653e8a4133626a772c31496fac7cf Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 11 May 2015 16:32:32 -0700
Subject: [PATCH 212/732] complete package-info and make get return null on 404

---
 .../gcloud/examples/StorageExample.java       |  9 ++--
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../google/gcloud/storage/StorageService.java |  5 +--
 .../gcloud/storage/StorageServiceImpl.java    | 41 +++++++++++++------
 .../google/gcloud/storage/package-info.java   | 12 +++++-
 5 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index b545e8323dcd..8b1a79b37212 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -69,7 +69,7 @@ private static abstract class StorageAction {
 
     abstract void run(StorageService storage, T request) throws Exception;
 
-    abstract T parse(String... args) throws IllegalArgumentException;
+    abstract T parse(String... args) throws IllegalArgumentException, IOException;
 
     protected String params() {
       return "";
@@ -115,6 +115,8 @@ public String params() {
   private static class InfoAction extends BlobsAction {
     @Override
     public void run(StorageService storage, Blob... blobs) {
+
+
       if (blobs.length == 1) {
         if (blobs[0].name().isEmpty()) {
           System.out.println(storage.get(blobs[0].bucket()));
@@ -217,13 +219,14 @@ public void run(StorageService storage, Tuple tuple) throws Exceptio
     }
 
     @Override
-    Tuple parse(String... args) {
+    Tuple parse(String... args) throws IOException {
       if (args.length < 2 || args.length > 3) {
         throw new IllegalArgumentException();
       }
       Path path = Paths.get(args[0]);
+      String contentType = Files.probeContentType(path);
       String blob = args.length < 3 ? path.getFileName().toString() : args[2];
-      return Tuple.of(path, Blob.of(args[1], blob));
+      return Tuple.of(path, Blob.builder(args[1], blob).contentType(contentType).build());
     }
 
     @Override
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 7ebbb1340d0f..455624f79b65 100644
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.AbstractInputStreamContent;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new AbstractInputStreamContent(storageObject.getContentType()) {
                @Override
                public InputStream getInputStream() throws IOException {
                  return new ByteArrayInputStream(content);
                }

                @Override
                public long getLength() throws IOException {
                  return content.length;
                }

                @Override
                public boolean retrySupported() {
                  return true;
                }
              })
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object content = option.get(options);
        if (content != null) {
          url.set(option.value(), content.toString());
        }
      }
      JsonFactory jsonFactory = storage.getJsonFactory();
      HttpRequestFactory requestFactory = storage.getRequestFactory();
      HttpRequest httpRequest =
          requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new ByteArrayContent(storageObject.getContentType(), content))
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object content = option.get(options);
        if (content != null) {
          url.set(option.value(), content.toString());
        }
      }
      JsonFactory jsonFactory = storage.getJsonFactory();
      HttpRequestFactory requestFactory = storage.getRequestFactory();
      HttpRequest httpRequest =
          requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 2294704eac33..ca130162d5cd 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -412,14 +412,14 @@ public static Builder builder() {
   Blob create(Blob blob, byte[] content, BlobTargetOption... options);
 
   /**
-   * Return the requested bucket.
+   * Return the requested bucket or {@code null} if not found.
    *
    * @throws StorageServiceException upon failure
    */
   Bucket get(String bucket, BucketSourceOption... options);
 
   /**
-   * Return the requested blob.
+   * Return the requested blob or {@code null} if not found.
    *
    * @throws StorageServiceException upon failure
    */
@@ -516,5 +516,4 @@ public static Builder builder() {
    * @throws StorageServiceException upon failure
    */
   BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
-
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index e577cf2c0aa3..b4f692f7e8cc 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.gcloud.RetryHelper.runWithRetries;
 import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
@@ -32,7 +33,6 @@
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Function;
 import com.google.common.base.Functions;
-import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
@@ -76,6 +76,7 @@ public RetryResult beforeEval(Exception exception) {
   };
   private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
       .abortOn(RuntimeException.class).interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
+  private static final byte[] EMPTY_BYTE_ARRAY = {};
 
   private final StorageRpc storageRpc;
   private final RetryParams retryParams;
@@ -83,7 +84,7 @@ public RetryResult beforeEval(Exception exception) {
   StorageServiceImpl(StorageServiceOptions options) {
     super(options);
     storageRpc = options.storageRpc();
-    retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
+    retryParams = firstNonNull(options.retryParams(), RetryParams.noRetries());
     // todo: replace nulls with Value.asNull (per toPb)
     // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors
     // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite
@@ -111,7 +112,7 @@ public Blob create(Blob blob, final byte[] content, BlobTargetOption... options)
     return Blob.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
-        return storageRpc.create(blobPb, content, optionsMap);
+        return storageRpc.create(blobPb, firstNonNull(content, EMPTY_BYTE_ARRAY), optionsMap);
       }
     }, retryParams, EXCEPTION_HANDLER));
   }
@@ -120,25 +121,41 @@ public StorageObject call() {
   public Bucket get(String bucket, BucketSourceOption... options) {
     final com.google.api.services.storage.model.Bucket bucketPb = Bucket.of(bucket).toPb();
     final Map optionsMap = optionMap(options);
-    return Bucket.fromPb(runWithRetries(
+    com.google.api.services.storage.model.Bucket answer = runWithRetries(
         new Callable() {
           @Override
           public com.google.api.services.storage.model.Bucket call() {
-            return storageRpc.get(bucketPb, optionsMap);
+            try {
+              return storageRpc.get(bucketPb, optionsMap);
+            } catch (StorageServiceException ex) {
+              if (ex.code() == 404) {
+                return null;
+              }
+              throw ex;
+            }
           }
-        }, retryParams, EXCEPTION_HANDLER));
+        }, retryParams, EXCEPTION_HANDLER);
+    return answer == null ? null : Bucket.fromPb(answer);
   }
 
   @Override
   public Blob get(String bucket, String blob, BlobSourceOption... options) {
     final StorageObject storedObject = Blob.of(bucket, blob).toPb();
     final Map optionsMap = optionMap(options);
-    return Blob.fromPb(runWithRetries(new Callable() {
+    StorageObject storageObject = runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
-        return storageRpc.get(storedObject, optionsMap);
+        try {
+          return storageRpc.get(storedObject, optionsMap);
+        } catch (StorageServiceException ex) {
+          if (ex.code() == 404) {
+            return null;
+          }
+          throw ex;
+        }
       }
-    }, retryParams, EXCEPTION_HANDLER));
+    }, retryParams, EXCEPTION_HANDLER);
+    return storageObject == null ? null : Blob.fromPb(storageObject);
   }
 
   @Override
@@ -368,7 +385,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
 
     private void initTransients() {
       storageRpc = serviceOptions.storageRpc();
-      retryParams = MoreObjects.firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries());
+      retryParams = firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries());
       storageObject = blob.toPb();
     }
 
@@ -504,7 +521,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
 
     private void initTransients() {
       storageRpc = options.storageRpc();
-      retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
+      retryParams = firstNonNull(options.retryParams(), RetryParams.noRetries());
       storageObject = blob.toPb();
     }
 
@@ -600,7 +617,7 @@ private static  void addToOptionMap(StorageRpc.Option getOption, StorageRpc.O
       T value = (T) map.remove(getOption);
       checkArgument(value != null || defaultValue != null,
           "Option " + getOption.value() + " is missing a value");
-      value = MoreObjects.firstNonNull(value, defaultValue);
+      value = firstNonNull(value, defaultValue);
       map.put(putOption, value);
     }
   }
diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/src/main/java/com/google/gcloud/storage/package-info.java
index e938755271eb..c02b743d0977 100644
--- a/src/main/java/com/google/gcloud/storage/package-info.java
+++ b/src/main/java/com/google/gcloud/storage/package-info.java
@@ -22,8 +22,16 @@
  * StorageServiceOptions options = StorageServiceOptions.builder().projectId("project").build();
  * StorageService storage = StorageServiceFactory.instance().get(options);
  * byte[] content = readContent();
- * Blob blob = storage.create(Blob.of("bucket", "blob_name"), content);
- * } 
    
+ * Blob blob = storage.get("bucket", "blob_name");
+ * if (blob == null) {
+ *   storage.create(Blob.of("bucket", "blob_name"), content);
+ * } else {
+ *   byte[] prevContent = storage.load("bucket", "blob_name");
+ *   content = mergeContent(prevContent, content);
+ *   WritableByteChannel channel = storage.writer(blob);
+ *   channel.write(ByteBuffer.wrap(content));
+ *   channel.close();
+ * }}
    
  *
  * @see Google Cloud Storage
  */

From 703575bea4c585001ab52751f8069243764ec957 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 11 May 2015 16:45:00 -0700
Subject: [PATCH 213/732] add another shortcut for CopyRequest

---
 src/main/java/com/google/gcloud/storage/StorageService.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index ca130162d5cd..4284f1517f10 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -390,6 +390,10 @@ public static CopyRequest of(String sourceBucket, String sourceBlob, Blob target
       return builder().source(sourceBucket, sourceBlob).target(target).build();
     }
 
+    public static CopyRequest of(String sourceBucket, String sourceBlob, String targetBlob) {
+      return of(sourceBucket, sourceBlob, Blob.of(sourceBucket, targetBlob));
+    }
+
     public static Builder builder() {
       return new Builder();
     }

From 502365303e2a961b61bf851f94a2ac40bf30513b Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 11 May 2015 16:49:12 -0700
Subject: [PATCH 214/732] add another shortcut for ComposeRequest

---
 src/main/java/com/google/gcloud/storage/StorageService.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 4284f1517f10..62c62ad8baaa 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -306,6 +306,10 @@ public static ComposeRequest of(Iterable sources, Blob target) {
       return builder().target(target).addSource(sources).build();
     }
 
+    public static ComposeRequest of(String bucket, Iterable sources, String target) {
+      return of(sources, Blob.of(bucket, target));
+    }
+
     public static Builder builder() {
       return new Builder();
     }

From 294cfb4e76c7df4de5fe0c52c0ff999dfbca33ea Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 11 May 2015 17:12:38 -0700
Subject: [PATCH 215/732] make batch gets return null as well on 404

---
 .../google/gcloud/storage/BatchResponse.java    | 16 ++++++++++++++--
 .../gcloud/storage/StorageServiceImpl.java      | 17 ++++++++++++++---
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index 0cce1300bcec..fab4d964e1c6 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -36,10 +36,12 @@ public class BatchResponse implements Serializable {
   public static class Result implements Serializable {
 
     private static final long serialVersionUID = -1946539570170529094L;
+    private static final Result EMPTY = new BatchResponse.Result(null);
 
     private final T value;
     private final StorageServiceException exception;
 
+
     Result(T value) {
       this.value = value;
       this.exception = null;
@@ -50,13 +52,15 @@ public static class Result implements Serializable {
       this.value = null;
     }
 
-
     /**
      * Returns the result.
      *
      * @throws StorageServiceException if failed
      */
     public T result() throws StorageServiceException {
+      if (failed()) {
+        throw failure();
+      }
       return value;
     }
 
@@ -76,7 +80,15 @@ public boolean failed() {
 
     @Override
     public String toString() {
-      return MoreObjects.firstNonNull(value, exception).toString();
+      return MoreObjects.toStringHelper(this)
+          .add("value", value)
+          .add("exception", exception)
+          .toString();
+    }
+
+    @SuppressWarnings("unchecked")
+    static  Result empty() {
+      return EMPTY;
     }
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index b4f692f7e8cc..42299f2bc6bd 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -37,6 +37,8 @@
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
 import com.google.gcloud.ExceptionHandler.Interceptor;
@@ -52,6 +54,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.Callable;
 
 final class StorageServiceImpl extends BaseService implements StorageService {
@@ -325,20 +328,28 @@ public BatchResponse apply(BatchRequest batchRequest) {
     List> updates = transformBatchResult(
         toUpdate, response.updates, Blob.FROM_PB_FUNCTION);
     List> gets = transformBatchResult(
-        toGet, response.gets, Blob.FROM_PB_FUNCTION);
+        toGet, response.gets, Blob.FROM_PB_FUNCTION, 404);
     return new BatchResponse(deletes, updates, gets);
   }
 
   private  List> transformBatchResult(
       Iterable>> request,
-      Map> results, Function transform) {
+      Map> results, Function transform,
+      int... nullOnErrorCodes) {
+    Set nullOnErrorCodesSet = Sets.newHashSet(Ints.asList(nullOnErrorCodes));
     List> response = Lists.newArrayListWithCapacity(results.size());
     for (Tuple tuple : request) {
       Tuple result = results.get(tuple.x());
       if (result.x() != null) {
         response.add(new BatchResponse.Result<>(transform.apply(result.x())));
       } else {
-        response.add(new BatchResponse.Result(result.y()));
+        StorageServiceException exception = result.y();
+        if (nullOnErrorCodesSet.contains(exception.code())) {
+          //noinspection unchecked
+          response.add(BatchResponse.Result.empty());
+        } else {
+          response.add(new BatchResponse.Result(result.y()));
+        }
       }
     }
     return response;

From 144b460342783e9977f5d148319b67c06af67fbb Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 12 May 2015 14:02:30 -0700
Subject: [PATCH 216/732] add metadata update to example and fix builder
 user-settable fields

---
 .../gcloud/examples/StorageExample.java       | 44 ++++++++++++++++++-
 .../java/com/google/gcloud/storage/Blob.java  | 28 +++++++-----
 .../com/google/gcloud/storage/Bucket.java     |  4 +-
 .../gcloud/storage/StorageServiceImpl.java    |  7 +--
 4 files changed, 66 insertions(+), 17 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 8b1a79b37212..453576683966 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -57,7 +57,7 @@
  * -Dexec.args="[] list []| info [ []]|
  *  download   [local_file]| upload   []|
  *  delete  +| cp    |
- *  compose  + "}
+ *  compose  + | update_metadata   [key=value]*"}
  * 
  * 
  */
@@ -336,6 +336,45 @@ public String params() {
     }
   }
 
+  private static class UpdateMetadata extends StorageAction>> {
+
+    @Override
+    public void run(StorageService storage, Tuple> tuple)
+        throws IOException {
+      Blob blob = storage.get(tuple.x().bucket(), tuple.x().name());
+      if (blob == null) {
+        System.out.println("No such object");
+        return;
+      }
+      blob = blob.toBuilder().metadata(tuple.y()).build();
+      System.out.println("before: " + blob);
+      System.out.println(storage.update(blob));
+    }
+
+    @Override
+    Tuple> parse(String... args) {
+      if (args.length < 2) {
+        throw new IllegalArgumentException();
+      }
+      Blob blob = Blob.of(args[0], args[1]);
+      Map metadata = new HashMap<>();
+      for (int i = 2; i < args.length; i++) {
+        int idx = args[i].indexOf('=');
+        if (idx < 0) {
+          metadata.put(args[i], "");
+        } else {
+          metadata.put(args[i].substring(0, idx), args[i].substring(idx + 1));
+        }
+      }
+      return Tuple.of(blob, metadata);
+    }
+
+    @Override
+    public String params() {
+      return "  [local_file]";
+    }
+  }
+
   static {
     ACTIONS.put("info", new InfoAction());
     ACTIONS.put("delete", new DeleteAction());
@@ -344,6 +383,7 @@ public String params() {
     ACTIONS.put("download", new DownloadAction());
     ACTIONS.put("cp", new CopyAction());
     ACTIONS.put("compose", new ComposeAction());
+    ACTIONS.put("update_metadata", new UpdateMetadata());
   }
 
   public static void printUsage() {
@@ -378,7 +418,7 @@ public static void main(String... args) throws Exception {
       args = Arrays.copyOfRange(args, 1, args.length);
     }
     if (action == null) {
-      System.out.println("Unrecognized action '" + args[1] + "'");
+      System.out.println("Unrecognized action.");
       printUsage();
       return;
     }
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index 49d763b0f65a..ffaec524c67d 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -127,12 +127,12 @@ public Builder contentType(String contentType) {
       return this;
     }
 
-    Builder contentDisposition(String contentDisposition) {
+    public Builder contentDisposition(String contentDisposition) {
       this.contentDisposition = contentDisposition;
       return this;
     }
 
-    Builder contentLanguage(String contentLanguage) {
+    public Builder contentLanguage(String contentLanguage) {
       this.contentLanguage = contentLanguage;
       return this;
     }
@@ -157,12 +157,12 @@ public Builder acl(List acl) {
       return this;
     }
 
-    public Builder owner(Acl.Entity owner) {
+    Builder owner(Acl.Entity owner) {
       this.owner = owner;
       return this;
     }
 
-    public Builder size(Long size) {
+    Builder size(Long size) {
       this.size = size;
       return this;
     }
@@ -177,7 +177,7 @@ Builder selfLink(String selfLink) {
       return this;
     }
 
-    Builder md5(String md5) {
+    public Builder md5(String md5) {
       this.md5 = md5;
       return this;
     }
@@ -187,7 +187,7 @@ public Builder crc32c(String crc32c) {
       return this;
     }
 
-    public Builder mediaLink(String mediaLink) {
+    Builder mediaLink(String mediaLink) {
       this.mediaLink = mediaLink;
       return this;
     }
@@ -197,22 +197,22 @@ public Builder metadata(Map metadata) {
       return this;
     }
 
-    public Builder generation(Long generation) {
+    Builder generation(Long generation) {
       this.generation = generation;
       return this;
     }
 
-    public Builder metageneration(Long metageneration) {
+    Builder metageneration(Long metageneration) {
       this.metageneration = metageneration;
       return this;
     }
 
-    public Builder deleteTime(Long deleteTime) {
+    Builder deleteTime(Long deleteTime) {
       this.deleteTime = deleteTime;
       return this;
     }
 
-    public Builder updateTime(Long updateTime) {
+    Builder updateTime(Long updateTime) {
       this.updateTime = updateTime;
       return this;
     }
@@ -365,7 +365,13 @@ public Builder toBuilder() {
 
   @Override
   public String toString() {
-    return MoreObjects.toStringHelper(this).add("bucket", bucket).add("name", name).toString();
+    return MoreObjects.toStringHelper(this)
+        .add("bucket", bucket)
+        .add("name", name)
+        .add("size", size)
+        .add("content-type", contentType)
+        .add("metadata", metadata)
+        .toString();
   }
 
   public static Blob of(String bucket, String name) {
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index f6be9a78b2c6..7f68704f9e6a 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -561,7 +561,9 @@ public Builder toBuilder() {
 
   @Override
   public String toString() {
-    return MoreObjects.toStringHelper(this).add("name", name).toString();
+    return MoreObjects.toStringHelper(this)
+        .add("name", name)
+        .toString();
   }
 
   public static Bucket of(String name) {
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 42299f2bc6bd..02bbb97a2fe5 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -28,6 +28,7 @@
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 import static java.util.concurrent.Executors.callable;
 
 import com.google.api.services.storage.model.StorageObject;
@@ -131,7 +132,7 @@ public com.google.api.services.storage.model.Bucket call() {
             try {
               return storageRpc.get(bucketPb, optionsMap);
             } catch (StorageServiceException ex) {
-              if (ex.code() == 404) {
+              if (ex.code() == HTTP_NOT_FOUND) {
                 return null;
               }
               throw ex;
@@ -151,7 +152,7 @@ public StorageObject call() {
         try {
           return storageRpc.get(storedObject, optionsMap);
         } catch (StorageServiceException ex) {
-          if (ex.code() == 404) {
+          if (ex.code() == HTTP_NOT_FOUND) {
             return null;
           }
           throw ex;
@@ -328,7 +329,7 @@ public BatchResponse apply(BatchRequest batchRequest) {
     List> updates = transformBatchResult(
         toUpdate, response.updates, Blob.FROM_PB_FUNCTION);
     List> gets = transformBatchResult(
-        toGet, response.gets, Blob.FROM_PB_FUNCTION, 404);
+        toGet, response.gets, Blob.FROM_PB_FUNCTION, HTTP_NOT_FOUND);
     return new BatchResponse(deletes, updates, gets);
   }
 

From 6757b9b4e89b00648e51be536d380d61efb8e49b Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 12 May 2015 19:26:24 -0700
Subject: [PATCH 217/732] add serialization test

---
 .../java/com/google/gcloud/storage/Acl.java   | 25 ++++++---
 .../google/gcloud/storage/BatchRequest.java   | 53 +++++++++++++------
 .../google/gcloud/storage/BatchResponse.java  | 34 +++++++++++-
 .../java/com/google/gcloud/storage/Blob.java  | 16 +++++-
 .../com/google/gcloud/storage/Bucket.java     | 14 +++++
 .../java/com/google/gcloud/storage/Cors.java  | 33 +++++++++++-
 .../com/google/gcloud/storage/ListResult.java | 18 ++++++-
 .../gcloud/storage/StorageServiceImpl.java    | 15 +++---
 .../gcloud/storage/SerializationTest.java     | 47 +++++++++++++++-
 9 files changed, 221 insertions(+), 34 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 138ba4f3fa3a..b5bb685334c1 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -20,6 +20,7 @@
 import com.google.api.services.storage.model.ObjectAccessControl;
 
 import java.io.Serializable;
+import java.util.Objects;
 
 /**
  *  Access Control List on for buckets or blobs.
@@ -59,6 +60,19 @@ protected String value() {
       return value;
     }
 
+    @Override
+    public int hashCode() {
+      return Objects.hash(type, value);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj == null || !getClass().isAssignableFrom(obj.getClass())) {
+        return false;
+      }
+      return Objects.equals(toPb(), ((Entity)obj).toPb());
+    }
+
     @Override
     public String toString() {
       return toPb();
@@ -91,13 +105,12 @@ static Entity fromPb(String entity) {
     }
   }
 
-  public static class Domain extends Entity {
+  public static final class Domain extends Entity {
 
     private static final long serialVersionUID = -3033025857280447253L;
 
     public Domain(String domain) {
       super(Type.DOMAIN, domain);
-
     }
 
     public String domain() {
@@ -105,7 +118,7 @@ public String domain() {
     }
   }
 
-  public static class Group extends Entity {
+  public static final class Group extends Entity {
 
     private static final long serialVersionUID = -1660987136294408826L;
 
@@ -118,7 +131,7 @@ public String email() {
     }
   }
 
-  public static class User extends Entity {
+  public static final class User extends Entity {
 
     private static final long serialVersionUID = 3076518036392737008L;
     private static final String ALL_USERS = "allUsers";
@@ -154,7 +167,7 @@ public static User ofAllAuthenticatedUsers() {
     }
   }
 
-  public static class Project extends Entity {
+  public static final class Project extends Entity {
 
     private static final long serialVersionUID = 7933776866530023027L;
 
@@ -180,7 +193,7 @@ public String projectId() {
     }
   }
 
-  public static class RawEntity extends Entity {
+  public static final class RawEntity extends Entity {
 
     private static final long serialVersionUID = 3966205614223053950L;
 
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 9fbee876cf45..6f62d5c51ae4 100644
--- a/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -17,51 +17,56 @@
 package com.google.gcloud.storage;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import com.google.gcloud.storage.StorageService.BlobSourceOption;
 import com.google.gcloud.storage.StorageService.BlobTargetOption;
 
 import java.io.Serializable;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * Google storage batch request.
  */
-public class BatchRequest implements Serializable {
+public final class BatchRequest implements Serializable {
 
   private static final long serialVersionUID = -1527992265939800345L;
 
-  private final Map toDelete;
-  private final Map toUpdate;
-  private final Map toGet;
+  private final Map> toDelete;
+  private final Map> toUpdate;
+  private final Map> toGet;
 
   public static class Builder {
 
-    private Map toDelete = new LinkedHashMap<>();
-    private Map toUpdate = new LinkedHashMap<>();
-    private Map toGet = new LinkedHashMap<>();
+    private Map> toDelete = new LinkedHashMap<>();
+    private Map> toUpdate = new LinkedHashMap<>();
+    private Map> toGet = new LinkedHashMap<>();
 
     private Builder() {}
 
     /**
      * Delete the given blob.
      */
-    public void delete(String bucket, String blob, BlobSourceOption... options) {
-      toDelete.put(Blob.of(bucket, blob), options);
+    public Builder delete(String bucket, String blob, BlobSourceOption... options) {
+      toDelete.put(Blob.of(bucket, blob), Lists.newArrayList(options));
+      return this;
     }
 
     /**
      * Update the given blob.
      */
-    public void update(Blob blob, BlobTargetOption... options) {
-      toUpdate.put(blob, options);
+    public Builder update(Blob blob, BlobTargetOption... options) {
+      toUpdate.put(blob, Lists.newArrayList(options));
+      return this;
     }
 
     /**
      * Retrieve metadata for the given blob.
      */
-    public void get(String bucket, String blob, BlobSourceOption... options) {
-      toGet.put(Blob.of(bucket, blob), options);
+    public Builder get(String bucket, String blob, BlobSourceOption... options) {
+      toGet.put(Blob.of(bucket, blob), Lists.newArrayList(options));
+      return this;
     }
 
     public BatchRequest build() {
@@ -75,15 +80,31 @@ private BatchRequest(Builder builder) {
     toGet = ImmutableMap.copyOf(builder.toGet);
   }
 
-  Map toDelete() {
+  @Override
+  public int hashCode() {
+    return Objects.hash(toDelete, toUpdate, toGet);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof BatchRequest)) {
+      return false;
+    }
+    BatchRequest other = (BatchRequest) obj;
+    return Objects.equals(toDelete, other.toDelete)
+        && Objects.equals(toUpdate, other.toUpdate)
+        && Objects.equals(toGet, other.toGet);
+  }
+
+  Map> toDelete() {
     return toDelete;
   }
 
-  Map toUpdate() {
+  Map> toUpdate() {
     return toUpdate;
   }
 
-  Map toGet() {
+  Map> toGet() {
     return toGet;
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index fab4d964e1c6..c45b0623eb9b 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -21,11 +21,12 @@
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Google Storage batch response.
  */
-public class BatchResponse implements Serializable {
+public final class BatchResponse implements Serializable {
 
   private static final long serialVersionUID = 1057416839397037706L;
 
@@ -78,6 +79,21 @@ public boolean failed() {
       return exception != null;
     }
 
+    @Override
+    public int hashCode() {
+      return Objects.hash(value, exception);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!(obj instanceof Result)) {
+        return false;
+      }
+      Result other = (Result) obj;
+      return Objects.equals(value, other.value)
+          && Objects.equals(exception, other.exception);
+    }
+
     @Override
     public String toString() {
       return MoreObjects.toStringHelper(this)
@@ -99,6 +115,22 @@ static  Result empty() {
     this.getResult = ImmutableList.copyOf(getResult);
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(deleteResult, updateResult, getResult);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof BatchResponse)) {
+      return false;
+    }
+    BatchResponse other = (BatchResponse) obj;
+    return Objects.equals(deleteResult, other.deleteResult)
+        && Objects.equals(updateResult, other.updateResult)
+        && Objects.equals(updateResult, other.updateResult);
+  }
+
   /**
    * Returns the results for the delete operations using the request order.
    */
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index ffaec524c67d..7ef81bf2b240 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -32,13 +32,14 @@
 import java.math.BigInteger;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * A Google Storage object.
  *
  * @see Concepts and Terminology
  */
-public class Blob implements Serializable {
+public final class Blob implements Serializable {
 
   private static final long serialVersionUID = 2228487739943277159L;
 
@@ -386,6 +387,19 @@ public static Builder builder(String bucket, String name) {
     return new Builder().bucket(bucket).name(name);
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(bucket, name);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof Blob)) {
+      return  false;
+    }
+    return Objects.equals(toPb(), ((Blob)obj).toPb());
+  }
+
   StorageObject toPb() {
     StorageObject storageObject = new StorageObject();
     if (acl != null) {
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 7f68704f9e6a..c36b24c80269 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -39,6 +39,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * A Google Storage bucket.
@@ -559,6 +560,19 @@ public Builder toBuilder() {
         .deleteRules(deleteRules);
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(name);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof Bucket)) {
+      return  false;
+    }
+    return Objects.equals(toPb(), ((Bucket) obj).toPb());
+  }
+
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index 366c8d9223bd..9cedbc0d3b4c 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -29,6 +29,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Cross-Origin Resource Sharing (CORS) configuration for a bucket.
@@ -60,7 +61,7 @@ public enum Method {
     ANY, GET, HEAD, PUT, POST, DELETE
   }
 
-  public static class Origin implements Serializable {
+  public static final class Origin implements Serializable {
 
     private static final long serialVersionUID = -4447958124895577993L;
     private static final String ANY_URI = "*";
@@ -91,6 +92,19 @@ public static Origin of(String value) {
       return new Origin(value);
     }
 
+    @Override
+    public int hashCode() {
+      return value.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!(obj instanceof Origin)) {
+        return false;
+      }
+      return value.equals(((Origin)obj).value);
+    }
+
     @Override
     public String toString() {
       return value();
@@ -166,6 +180,23 @@ public Builder toBuilder() {
         .responseHeaders(responseHeaders);
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(maxAgeSeconds, methods, origins, responseHeaders);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof Cors)) {
+      return false;
+    }
+    Cors other = (Cors) obj;
+    return Objects.equals(maxAgeSeconds, other.maxAgeSeconds)
+        && Objects.equals(methods, other.methods)
+        && Objects.equals(origins, other.origins)
+        && Objects.equals(responseHeaders, other.responseHeaders);
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/src/main/java/com/google/gcloud/storage/ListResult.java b/src/main/java/com/google/gcloud/storage/ListResult.java
index 180e5f7a4155..dd843020376e 100644
--- a/src/main/java/com/google/gcloud/storage/ListResult.java
+++ b/src/main/java/com/google/gcloud/storage/ListResult.java
@@ -18,11 +18,12 @@
 
 import java.io.Serializable;
 import java.util.Iterator;
+import java.util.Objects;
 
 /**
  * Google Cloud storage list result.
  */
-public class ListResult implements Iterable, Serializable {
+public final class ListResult implements Iterable, Serializable {
 
   private static final long serialVersionUID = -6937287874908527950L;
 
@@ -42,4 +43,19 @@ public String nextPageCursor() {
   public Iterator iterator() {
     return results.iterator();
   }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(cursor, results);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof ListResult)) {
+      return false;
+    }
+    ListResult other = (ListResult) obj;
+    return Objects.equals(cursor, other.cursor)
+        && Objects.equals(results, other.results);
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 02bbb97a2fe5..1f66401105d0 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -302,24 +302,27 @@ public byte[] call() {
   public BatchResponse apply(BatchRequest batchRequest) {
     List>> toDelete =
         Lists.newArrayListWithCapacity(batchRequest.toDelete().size());
-    for (Map.Entry entry : batchRequest.toDelete().entrySet()) {
+    for (Map.Entry> entry : batchRequest.toDelete().entrySet()) {
       Blob blob = entry.getKey();
-      Map optionsMap = optionMap(blob, entry.getValue());
+      Map optionsMap =
+          optionMap(blob.generation(), blob.metageneration(), entry.getValue());
       StorageObject storageObject = blob.toPb();
       toDelete.add(Tuple.>of(storageObject, optionsMap));
     }
     List>> toUpdate =
         Lists.newArrayListWithCapacity(batchRequest.toUpdate().size());
-    for (Map.Entry entry : batchRequest.toUpdate().entrySet()) {
+    for (Map.Entry> entry : batchRequest.toUpdate().entrySet()) {
       Blob blob = entry.getKey();
-      Map optionsMap = optionMap(blob, entry.getValue());
+      Map optionsMap =
+          optionMap(blob.generation(), blob.metageneration(), entry.getValue());
       toUpdate.add(Tuple.>of(blob.toPb(), optionsMap));
     }
     List>> toGet =
         Lists.newArrayListWithCapacity(batchRequest.toGet().size());
-    for (Map.Entry entry : batchRequest.toGet().entrySet()) {
+    for (Map.Entry> entry : batchRequest.toGet().entrySet()) {
       Blob blob = entry.getKey();
-      Map optionsMap = optionMap(blob, entry.getValue());
+      Map optionsMap =
+          optionMap(blob.generation(), blob.metageneration(), entry.getValue());
       toGet.add(Tuple.>of(blob.toPb(), optionsMap));
     }
     StorageRpc.BatchResponse response =
diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/src/test/java/com/google/gcloud/storage/SerializationTest.java
index 7e3cb3e89258..365462d3f69a 100644
--- a/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -17,9 +17,11 @@
 package com.google.gcloud.storage;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 
 import com.google.gcloud.AuthCredentials;
 import com.google.gcloud.RetryParams;
+import com.google.gcloud.storage.Acl.Project.ProjectRole;
 
 import org.junit.Test;
 
@@ -28,9 +30,40 @@
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.Collections;
 
 public class SerializationTest {
 
+  private static final Acl.Domain ACL_DOMAIN = new Acl.Domain("domain");
+  private static final Acl.Group ACL_GROUP = new Acl.Group("group");
+  private static final Acl.Project ACL_PROJECT_ = new Acl.Project(ProjectRole.VIEWERS, "pid");
+  private static final Acl.User ACL_USER = new Acl.User("user");
+  private static final Acl.RawEntity ACL_RAW = new Acl.RawEntity("raw");
+  private static final Blob BLOB = Blob.of("b", "n");
+  private static final Bucket BUCKET = Bucket.of("b");
+  private static final Cors.Origin ORIGIN = Cors.Origin.any();
+  private static final Cors CORS =
+      Cors.builder().maxAgeSeconds(1).origins(Collections.singleton(ORIGIN)).build();
+  private static final BatchRequest BATCH_REQUEST = BatchRequest.builder().delete("B", "N").build();
+  private static final BatchResponse BATCH_RESPONSE = new BatchResponse(
+      Collections.singletonList(new BatchResponse.Result<>(true)),
+      Collections.>emptyList(),
+      Collections.>emptyList());
+  private static final ListResult LIST_RESULT =
+      new ListResult<>("c", Collections.singletonList(Blob.of("b", "n")));
+  private static StorageService.BlobListOption BLOB_LIST_OPTIONS =
+      StorageService.BlobListOption.maxResults(100);
+  private static StorageService.BlobSourceOption BLOB_SOURCE_OPTIONS =
+      StorageService.BlobSourceOption.generationMatch(1);
+  private static StorageService.BlobTargetOption BLOB_TARGET_OPTIONS =
+      StorageService.BlobTargetOption.generationMatch();
+  private static StorageService.BucketListOption BUCKET_LIST_OPTIONS =
+      StorageService.BucketListOption.prefix("bla");
+  private static StorageService.BucketSourceOption BUCKET_SOURCE_OPTIONS =
+      StorageService.BucketSourceOption.metagenerationMatch(1);
+  private static StorageService.BucketTargetOption BUCKET_TARGET_OPTIONS =
+      StorageService.BucketTargetOption.metagenerationNotMatch();
 
   @Test
   public void testServiceOptions() throws Exception {
@@ -52,8 +85,18 @@ public void testServiceOptions() throws Exception {
   }
 
   @Test
-  public void testTypes() throws Exception {
-    // todo: implement
+  public void testModelAndRequests() throws Exception {
+    Serializable[] objects = {ACL_DOMAIN, ACL_GROUP, ACL_PROJECT_, ACL_USER, ACL_RAW, BLOB, BUCKET,
+      ORIGIN, CORS, BATCH_REQUEST,BATCH_RESPONSE, LIST_RESULT, BLOB_LIST_OPTIONS,
+        BLOB_SOURCE_OPTIONS, BLOB_TARGET_OPTIONS, BUCKET_LIST_OPTIONS, BUCKET_SOURCE_OPTIONS,
+        BUCKET_TARGET_OPTIONS};
+    for (Serializable obj : objects) {
+      Object copy = serializeAndDeserialize(obj);
+      assertEquals(obj, obj);
+      assertEquals(obj, copy);
+      assertNotSame(obj, copy);
+      assertEquals(copy, copy);
+    }
   }
 
   @SuppressWarnings("unchecked")

From 4a45c87911120864a44e65af2c7c49ee88d7ca9a Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 12 May 2015 22:02:20 -0700
Subject: [PATCH 218/732] add comments to example

---
 .../gcloud/examples/StorageExample.java       | 133 ++++++++++++++----
 .../google/gcloud/storage/BatchResponse.java  |   2 +-
 2 files changed, 108 insertions(+), 27 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 453576683966..05f1f9336107 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -46,7 +46,7 @@
 /**
  * An example of using the Google Cloud Storage.
  * 
    
- * This example demonstrates a simple/typical usage.
+ * This example demonstrates a simple/typical storage usage.
  * 
    
  * Steps needed for running the example:
  * 
      
@@ -60,6 +60,11 @@
  *  compose  + | update_metadata   [key=value]*"}
  * 
  * 
    
+ *
+ * The first parameter is an optional project_id (logged-in project will be used if not supplied).
+ * Second parameter is a Storage operation (list, delete, compose,...) to demonstrate the its
+ * usage. Any other arguments are specific to the operation.
+ * See each action's run method for the specific Storage interaction.
  */
 public class StorageExample {
 
@@ -112,24 +117,34 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how to retrieve Bucket or Blob metadata.
+   * If more than one blob is supplied a Batch operation would be used to get all blobs metadata
+   * in a single RPC.
+   */
   private static class InfoAction extends BlobsAction {
     @Override
     public void run(StorageService storage, Blob... blobs) {
-
-
       if (blobs.length == 1) {
         if (blobs[0].name().isEmpty()) {
-          System.out.println(storage.get(blobs[0].bucket()));
+          // get Bucket
+          Bucket bucket = storage.get(blobs[0].bucket());
+          System.out.println("Bucket info: " + bucket);
         } else {
-          System.out.println(storage.get(blobs[0].bucket(), blobs[0].name()));
+          // get Blob
+          Blob blob = storage.get(blobs[0].bucket(), blobs[0].name());
+          System.out.println("Blob info: " + blob);
         }
       } else {
+        // use batch to get multiple blobs.
         BatchRequest.Builder batch = BatchRequest.builder();
         for (Blob blob : blobs) {
           batch.get(blob.bucket(), blob.name());
         }
         BatchResponse response = storage.apply(batch.build());
-        System.out.println(response.gets());
+        for (BatchResponse.Result result : response.gets()) {
+          System.out.println(result.get());
+        }
       }
     }
 
@@ -147,22 +162,41 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how to delete a blob.
+   * If more than one blob is supplied a Batch operation would be used to delete all requested
+   * blobs in a single RPC.
+   */
   private static class DeleteAction extends BlobsAction {
     @Override
     public void run(StorageService storage, Blob... blobs) {
       if (blobs.length == 1) {
-        System.out.println(storage.delete(blobs[0].bucket(), blobs[0].name()));
+        boolean wasDeleted = storage.delete(blobs[0].bucket(), blobs[0].name());
+        if (wasDeleted) {
+          System.out.println("Blob " + blobs[0] + " was deleted");
+        }
       } else {
+        // use batch operation
         BatchRequest.Builder batch = BatchRequest.builder();
         for (Blob blob : blobs) {
           batch.delete(blob.bucket(), blob.name());
         }
+        int index = 0;
         BatchResponse response = storage.apply(batch.build());
-        System.out.println(response.deletes());
+        for (BatchResponse.Result result : response.deletes()) {
+          if (result.get()) {
+            // request order is maintained
+            System.out.println("Blob " + blobs[index] + " was deleted");
+          }
+          index++;
+        }
       }
     }
   }
 
+  /**
+   * This class demonstrates how to list buckets or a bucket's blobs.
+   */
   private static class ListAction extends StorageAction {
 
     @Override
@@ -179,10 +213,12 @@ String parse(String... args) {
     @Override
     public void run(StorageService storage, String bucket) {
       if (bucket == null) {
+        // list buckets
         for (Bucket b : storage.list()) {
           System.out.println(b);
         }
       } else {
+        // list a bucket's blobs
         for (Blob b : storage.list(bucket)) {
           System.out.println(b);
         }
@@ -195,13 +231,22 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how to create a new Blob or to update its content.
+   */
   private static class UploadAction extends StorageAction> {
     @Override
     public void run(StorageService storage, Tuple tuple) throws Exception {
-      if (Files.size(tuple.x()) > 1024) {
-        try (BlobWriteChannel writer = storage.writer(tuple.y())) {
+      run(storage, tuple.x(), tuple.y());
+    }
+
+    private void run(StorageService storage, Path uploadFrom, Blob blob) throws IOException {
+      if (Files.size(uploadFrom) > 1_000_000) {
+        // When content is not available or large (1MB or more) it is recommended
+        // to write it in chunks via the blob's channel writer.
+        try (BlobWriteChannel writer = storage.writer(blob)) {
           byte[] buffer = new byte[1024];
-          try (InputStream input = Files.newInputStream(tuple.x())) {
+          try (InputStream input = Files.newInputStream(uploadFrom)) {
             int limit;
             while ((limit = input.read(buffer)) >= 0) {
               try {
@@ -213,9 +258,11 @@ public void run(StorageService storage, Tuple tuple) throws Exceptio
           }
         }
       } else {
-        byte[] bytes = Files.readAllBytes(tuple.x());
-        System.out.println(storage.create(tuple.y(), bytes));
+        byte[] bytes = Files.readAllBytes(uploadFrom);
+        // create the blob in one request.
+        storage.create(blob, bytes);
       }
+      System.out.println("Blob was created");
     }
 
     @Override
@@ -235,22 +282,35 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how read a blob's content.
+   * The example will dump the content to a local file if one was given or write
+   * it to stdout otherwise.
+   */
   private static class DownloadAction extends StorageAction> {
 
     @Override
     public void run(StorageService storage, Tuple tuple) throws IOException {
-      Blob blob = storage.get(tuple.x().bucket(), tuple.x().name());
+      run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y());
+    }
+
+    private void run(StorageService storage, String bucket, String blobName, Path downloadTo)
+        throws IOException {
+      Blob blob = storage.get(bucket, blobName);
       if (blob == null) {
         System.out.println("No such object");
         return;
       }
       PrintStream writeTo = System.out;
-      if (tuple.y() != null) {
-        writeTo = new PrintStream(new FileOutputStream(tuple.y().toFile()));
+      if (downloadTo != null) {
+        writeTo = new PrintStream(new FileOutputStream(downloadTo.toFile()));
       }
-      if (blob.size() < 1024) {
-        writeTo.write(storage.load(blob.bucket(), blob.name()));
+      if (blob.size() < 1_000_000) {
+        // Blob is small read all its content in one request
+        byte[] content = storage.load(blob.bucket(), blob.name());
+        writeTo.write(content);
       } else {
+        // When Blob size is big or unknown use the blob's channel reader.
         try (BlobReadChannel reader = storage.reader(blob.bucket(), blob.name())) {
           WritableByteChannel channel = Channels.newChannel(writeTo);
           ByteBuffer bytes = ByteBuffer.allocate(64 * 1024);
@@ -261,7 +321,7 @@ public void run(StorageService storage, Tuple tuple) throws IOExcept
           }
         }
       }
-      if (tuple.y() == null) {
+      if (downloadTo == null) {
         writeTo.println();
       } else {
         writeTo.close();
@@ -291,10 +351,16 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how to use the copy command.
+   *
+   * @see Object copy
+   */
   private static class CopyAction extends StorageAction {
     @Override
     public void run(StorageService storage, CopyRequest request) {
-      System.out.println(storage.copy(request));
+      Blob copiedBlob = storage.copy(request);
+      System.out.println("Copied " + copiedBlob);
     }
 
     @Override
@@ -311,10 +377,16 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how to use the compose command.
+   *
+   * @see Object compose
+   */
   private static class ComposeAction extends StorageAction {
     @Override
     public void run(StorageService storage, ComposeRequest request) {
-      System.out.println(storage.compose(request));
+      Blob composedBlob = storage.compose(request);
+      System.out.println("Composed " + composedBlob);
     }
 
     @Override
@@ -336,19 +408,28 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how to update a blob's metadata.
+   *
+   * @see Object compose
+   */
   private static class UpdateMetadata extends StorageAction>> {
 
     @Override
     public void run(StorageService storage, Tuple> tuple)
         throws IOException {
-      Blob blob = storage.get(tuple.x().bucket(), tuple.x().name());
+      run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y());
+    }
+
+    private void run(StorageService storage, String bucket, String blobName,
+        Map metadata) {
+      Blob blob = storage.get(bucket, blobName);
       if (blob == null) {
         System.out.println("No such object");
         return;
       }
-      blob = blob.toBuilder().metadata(tuple.y()).build();
-      System.out.println("before: " + blob);
-      System.out.println(storage.update(blob));
+      blob = storage.update(blob.toBuilder().metadata(metadata).build());
+      System.out.println("Updated " + blob);
     }
 
     @Override
@@ -357,7 +438,7 @@ Tuple> parse(String... args) {
         throw new IllegalArgumentException();
       }
       Blob blob = Blob.of(args[0], args[1]);
-      Map metadata = new HashMap<>();
+      Map metadata = new HashMap<>();
       for (int i = 2; i < args.length; i++) {
         int idx = args[i].indexOf('=');
         if (idx < 0) {
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index c45b0623eb9b..f0675e348f72 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -58,7 +58,7 @@ public static class Result implements Serializable {
      *
      * @throws StorageServiceException if failed
      */
-    public T result() throws StorageServiceException {
+    public T get() throws StorageServiceException {
       if (failed()) {
         throw failure();
       }

From 28db1f72d290de1155f00d60df8fb91fd843ffe5 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 13 May 2015 16:28:29 -0700
Subject: [PATCH 219/732] Replace given null values with Apiary nulls and
 provide doesNotExists option

---
 .../java/com/google/gcloud/storage/Blob.java  | 42 ++++++++++---------
 .../com/google/gcloud/storage/Bucket.java     | 22 ++++++----
 .../google/gcloud/storage/StorageService.java |  4 ++
 3 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/src/main/java/com/google/gcloud/storage/Blob.java
index 7ef81bf2b240..31b88981187d 100644
--- a/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/src/main/java/com/google/gcloud/storage/Blob.java
@@ -16,8 +16,10 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.api.client.util.Data;
 import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.ObjectAccessControl;
 import com.google.api.services.storage.model.StorageObject;
@@ -124,22 +126,22 @@ public Builder name(String name) {
     }
 
     public Builder contentType(String contentType) {
-      this.contentType = contentType;
+      this.contentType = firstNonNull(contentType, Data.nullOf(String.class));
       return this;
     }
 
     public Builder contentDisposition(String contentDisposition) {
-      this.contentDisposition = contentDisposition;
+      this.contentDisposition = firstNonNull(contentDisposition, Data.nullOf(String.class));
       return this;
     }
 
     public Builder contentLanguage(String contentLanguage) {
-      this.contentLanguage = contentLanguage;
+      this.contentLanguage = firstNonNull(contentLanguage, Data.nullOf(String.class));
       return this;
     }
 
     public Builder contentEncoding(String contentEncoding) {
-      this.contentEncoding = contentEncoding;
+      this.contentEncoding = firstNonNull(contentEncoding, Data.nullOf(String.class));
       return this;
     }
 
@@ -149,7 +151,7 @@ Builder componentCount(Integer componentCount) {
     }
 
     public Builder cacheControl(String cacheControl) {
-      this.cacheControl = cacheControl;
+      this.cacheControl = firstNonNull(cacheControl, Data.nullOf(String.class));
       return this;
     }
 
@@ -179,12 +181,12 @@ Builder selfLink(String selfLink) {
     }
 
     public Builder md5(String md5) {
-      this.md5 = md5;
+      this.md5 = firstNonNull(md5, Data.nullOf(String.class));
       return this;
     }
 
     public Builder crc32c(String crc32c) {
-      this.crc32c = crc32c;
+      this.crc32c = firstNonNull(crc32c, Data.nullOf(String.class));
       return this;
     }
 
@@ -263,7 +265,7 @@ public String name() {
   }
 
   public String cacheControl() {
-    return cacheControl;
+    return Data.isNull(cacheControl) ? null : cacheControl;
   }
 
   public List acl() {
@@ -279,19 +281,19 @@ public Long size() {
   }
 
   public String contentType() {
-    return contentType;
+    return Data.isNull(contentType) ? null : contentType;
   }
 
   public String contentEncoding() {
-    return contentEncoding;
+    return Data.isNull(contentEncoding) ? null : contentEncoding;
   }
 
   public String contentDisposition() {
-    return contentDisposition;
+    return Data.isNull(contentDisposition) ? null : contentDisposition;
   }
 
   public String contentLanguage() {
-    return contentEncoding;
+    return Data.isNull(contentLanguage) ? null : contentLanguage;
   }
 
   public Integer componentCount() {
@@ -307,11 +309,11 @@ public String selfLink() {
   }
 
   public String md5() {
-    return md5;
+    return Data.isNull(md5) ? null : md5;
   }
 
   public String crc32c() {
-    return crc32c;
+    return Data.isNull(crc32c) ? null : crc32c;
   }
 
   public String mediaLink() {
@@ -367,11 +369,11 @@ public Builder toBuilder() {
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
-        .add("bucket", bucket)
-        .add("name", name)
-        .add("size", size)
-        .add("content-type", contentType)
-        .add("metadata", metadata)
+        .add("bucket", bucket())
+        .add("name", name())
+        .add("size", size())
+        .add("content-type", contentType())
+        .add("metadata", metadata())
         .toString();
   }
 
@@ -397,7 +399,7 @@ public boolean equals(Object obj) {
     if (!(obj instanceof Blob)) {
       return  false;
     }
-    return Objects.equals(toPb(), ((Blob)obj).toPb());
+    return Objects.equals(toPb(), ((Blob) obj).toPb());
   }
 
   StorageObject toPb() {
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index c36b24c80269..793ae0199d5c 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -17,9 +17,11 @@
 package com.google.gcloud.storage;
 
 import static com.google.api.client.repackaged.com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.Lists.transform;
 
 import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.util.Data;
 import com.google.api.client.util.DateTime;
 import com.google.api.services.storage.model.Bucket.Lifecycle;
 import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
@@ -67,7 +69,6 @@ public final class Bucket implements Serializable {
   private final Location location;
   private final StorageClass storageClass;
 
-
   static final Function FROM_PB_FUNCTION =
       new Function() {
         @Override
@@ -248,6 +249,9 @@ public static final class StorageClass implements Serializable {
 
     private static final long serialVersionUID = 374002156285326563L;
     private static final ImmutableMap STRING_TO_OPTION;
+    private static final StorageClass NULL_VALUE =
+        new StorageClass(Data.nullOf(String.class));
+
     private final String value;
 
     public enum Option {
@@ -299,6 +303,8 @@ public static final class Location implements Serializable {
 
     private static final long serialVersionUID = 9073107666838637662L;
     private static final ImmutableMap STRING_TO_OPTION;
+    private static final Location NULL_VALUE = new Location(Data.nullOf(String.class));
+
     private final String value;
 
     public enum Option {
@@ -392,7 +398,7 @@ Builder selfLink(String selfLink) {
     }
 
     public Builder versioningEnabled(Boolean enable) {
-      this.versioningEnabled = enable;
+      this.versioningEnabled = firstNonNull(enable, Data.nullOf(Boolean.class));
       return this;
     }
 
@@ -412,12 +418,12 @@ public Builder deleteRules(Iterable rules) {
     }
 
     public Builder storageClass(StorageClass storageClass) {
-      this.storageClass = storageClass;
+      this.storageClass = firstNonNull(storageClass, StorageClass.NULL_VALUE);
       return this;
     }
 
     public Builder location(Location location) {
-      this.location = location;
+      this.location = firstNonNull(location, Location.NULL_VALUE);
       return this;
     }
 
@@ -493,7 +499,7 @@ public String selfLink() {
   }
 
   public Boolean versioningEnabled() {
-    return versioningEnabled;
+    return Data.isNull(versioningEnabled) ? null : versioningEnabled;
   }
 
   public String indexPage() {
@@ -521,11 +527,11 @@ public Long metageneration() {
   }
 
   public Location location() {
-    return location;
+    return location == null || Data.isNull(location.value) ? null : location;
   }
 
   public StorageClass storageClass() {
-    return storageClass;
+    return storageClass == null || Data.isNull(storageClass.value) ? null : storageClass;
   }
 
   public List cors() {
@@ -576,7 +582,7 @@ public boolean equals(Object obj) {
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
-        .add("name", name)
+        .add("name", name())
         .toString();
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 62c62ad8baaa..9e95b0f1281d 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -121,6 +121,10 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
       return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
 
+    public static BlobTargetOption doesNotExists() {
+      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0);
+    }
+
     public static BlobTargetOption generationMatch() {
       return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH);
     }

From f409c22be8f5d8bb77f3c35d5e629a0967a60393 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 13 May 2015 17:45:56 -0700
Subject: [PATCH 220/732] add apiary references to all example actions

---
 .../google/gcloud/examples/StorageExample.java   | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index 05f1f9336107..b6f54efa1d93 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -121,6 +121,8 @@ public String params() {
    * This class demonstrates how to retrieve Bucket or Blob metadata.
    * If more than one blob is supplied a Batch operation would be used to get all blobs metadata
    * in a single RPC.
+   *
+   * @see Objects: get
    */
   private static class InfoAction extends BlobsAction {
     @Override
@@ -166,6 +168,8 @@ public String params() {
    * This class demonstrates how to delete a blob.
    * If more than one blob is supplied a Batch operation would be used to delete all requested
    * blobs in a single RPC.
+   *
+   * @see Objects: delete
    */
   private static class DeleteAction extends BlobsAction {
     @Override
@@ -196,6 +200,8 @@ public void run(StorageService storage, Blob... blobs) {
 
   /**
    * This class demonstrates how to list buckets or a bucket's blobs.
+   *
+   * @see Objects: list
    */
   private static class ListAction extends StorageAction {
 
@@ -233,6 +239,8 @@ public String params() {
 
   /**
    * This class demonstrates how to create a new Blob or to update its content.
+   *
+   * @see Objects: insert
    */
   private static class UploadAction extends StorageAction> {
     @Override
@@ -286,6 +294,8 @@ public String params() {
    * This class demonstrates how read a blob's content.
    * The example will dump the content to a local file if one was given or write
    * it to stdout otherwise.
+   *
+   * @see Objects: get
    */
   private static class DownloadAction extends StorageAction> {
 
@@ -354,7 +364,7 @@ public String params() {
   /**
    * This class demonstrates how to use the copy command.
    *
-   * @see Object copy
+   * @see Objects: copy
    */
   private static class CopyAction extends StorageAction {
     @Override
@@ -380,7 +390,7 @@ public String params() {
   /**
    * This class demonstrates how to use the compose command.
    *
-   * @see Object compose
+   * @see Objects: compose
    */
   private static class ComposeAction extends StorageAction {
     @Override
@@ -411,7 +421,7 @@ public String params() {
   /**
    * This class demonstrates how to update a blob's metadata.
    *
-   * @see Object compose
+   * @see Objects: update
    */
   private static class UpdateMetadata extends StorageAction>> {
 

From 2dd990c18e4b577f39a51e0d66a31a57af643bac Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 13 May 2015 19:02:22 -0700
Subject: [PATCH 221/732] acl test

---
 .../java/com/google/gcloud/storage/Acl.java   | 40 ++++++++
 .../com/google/gcloud/storage/AclTest.java    | 94 +++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 src/test/java/com/google/gcloud/storage/AclTest.java

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index 138ba4f3fa3a..7e78bd7981d6 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -20,6 +20,7 @@
 import com.google.api.services.storage.model.ObjectAccessControl;
 
 import java.io.Serializable;
+import java.util.Objects;
 
 /**
  *  Access Control List on for buckets or blobs.
@@ -59,6 +60,24 @@ protected String value() {
       return value;
     }
 
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      Entity entity = (Entity) o;
+      return Objects.equals(type, entity.type) &&
+          Objects.equals(value, entity.value);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(type, value);
+    }
+
     @Override
     public String toString() {
       return toPb();
@@ -81,6 +100,9 @@ static Entity fromPb(String entity) {
       if (entity.startsWith("group-")) {
         return new Group(entity.substring(6));
       }
+      if (entity.startsWith("domain-")) {
+        return new Domain(entity.substring(7));
+      }
       if (entity.startsWith("project-")) {
         int idx = entity.indexOf('-', 8);
         String team = entity.substring(8, idx);
@@ -207,6 +229,24 @@ public Role role() {
     return role;
   }
 
+  @Override
+  public int hashCode() {
+    return Objects.hash(entity, role);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    final Acl other = (Acl) obj;
+    return Objects.equals(this.entity, other.entity)
+        && Objects.equals(this.role, other.role);
+  }
+
   BucketAccessControl toBucketPb() {
     BucketAccessControl bucketPb = new BucketAccessControl();
     bucketPb.setRole(role().toString());
diff --git a/src/test/java/com/google/gcloud/storage/AclTest.java b/src/test/java/com/google/gcloud/storage/AclTest.java
new file mode 100644
index 000000000000..6a11fb0b2810
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/AclTest.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static org.junit.Assert.assertEquals;
+
+import com.google.api.services.storage.model.BucketAccessControl;
+import com.google.api.services.storage.model.ObjectAccessControl;
+import com.google.gcloud.storage.Acl.Domain;
+import com.google.gcloud.storage.Acl.Entity;
+import com.google.gcloud.storage.Acl.Entity.Type;
+import com.google.gcloud.storage.Acl.Group;
+import com.google.gcloud.storage.Acl.Project;
+import com.google.gcloud.storage.Acl.Project.ProjectRole;
+import com.google.gcloud.storage.Acl.RawEntity;
+import com.google.gcloud.storage.Acl.Role;
+import com.google.gcloud.storage.Acl.User;
+
+import org.junit.Test;
+
+public class AclTest {
+
+  @Test
+  public void testDomainEntity() {
+    Domain acl = new Domain("d1");
+    assertEquals("d1", acl.domain());
+    assertEquals(Type.DOMAIN, acl.type());
+    String pb = acl.toPb();
+    assertEquals(acl, Entity.fromPb(pb));
+  }
+
+  @Test
+  public void testGroupEntity() {
+    Group acl = new Group("g1");
+    assertEquals("g1", acl.email());
+    assertEquals(Type.GROUP, acl.type());
+    String pb = acl.toPb();
+    assertEquals(acl, Entity.fromPb(pb));
+  }
+
+  @Test
+  public void testUserEntity() {
+    User acl = new User("u1");
+    assertEquals("u1", acl.email());
+    assertEquals(Type.USER, acl.type());
+    String pb = acl.toPb();
+    assertEquals(acl, Entity.fromPb(pb));
+  }
+
+  @Test
+  public void testProjectEntity() {
+    Project acl = new Project(ProjectRole.VIEWERS, "p1");
+    assertEquals(ProjectRole.VIEWERS, acl.projectRole());
+    assertEquals("p1", acl.projectId());
+    assertEquals(Type.PROJECT, acl.type());
+    String pb = acl.toPb();
+    assertEquals(acl, Entity.fromPb(pb));
+  }
+
+  @Test
+  public void testRawEntity() {
+    Entity acl = new RawEntity("bla");
+    assertEquals("bla", acl.value());
+    assertEquals(Type.UNKNOWN, acl.type());
+    String pb = acl.toPb();
+    assertEquals(acl, Entity.fromPb(pb));
+  }
+
+
+  @Test
+  public void testAcl() {
+    Acl acl = new Acl(User.ofAllUsers(), Role.READER);
+    assertEquals(User.ofAllUsers(), acl.entity());
+    assertEquals(Role.READER, acl.role());
+    ObjectAccessControl objectPb = acl.toObjectPb();
+    assertEquals(acl, Acl.fromPb(objectPb));
+    BucketAccessControl bucketPb = acl.toBucketPb();
+    assertEquals(acl, Acl.fromPb(bucketPb));
+  }
+}

From 2c8387df7f7577062982456cdfb8b1391d2b5772 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 14 May 2015 10:08:43 -0700
Subject: [PATCH 222/732] adding BatchRequestTest

---
 .../gcloud/storage/BatchRequestTest.java      | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 src/test/java/com/google/gcloud/storage/BatchRequestTest.java

diff --git a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
new file mode 100644
index 000000000000..d459ba73a0af
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import com.google.gcloud.storage.BatchRequest.Builder;
+import com.google.gcloud.storage.StorageService.BlobSourceOption;
+
+import org.junit.Test;
+
+public class BatchRequestTest {
+
+  @Test
+  public void testRequest() throws Exception {
+    Builder builder = BatchRequest.builder();
+    builder.delete("b1", "o1")
+    BatchRequest.builder().delete("b1", "o1",
+        BlobSourceOption.generationMatch(), BlobSourceOption.metagenerationMatch());
+  }
+
+  @Test
+  public void testToUpdate() throws Exception {
+
+  }
+
+  @Test
+  public void testToGet() throws Exception {
+
+  }
+
+  @Test
+  public void testBuilder() throws Exception {
+
+  }
+}

From 5ce009facdac9595aad7be72394c3dd628f34c9e Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 14 May 2015 17:33:08 -0700
Subject: [PATCH 223/732] work in progress

---
 .../gcloud/storage/BatchRequestTest.java      | 55 +++++++++++++------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
index d459ba73a0af..56c1f72dd84d 100644
--- a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
+++ b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
@@ -16,33 +16,54 @@
 
 package com.google.gcloud.storage;
 
-import com.google.gcloud.storage.BatchRequest.Builder;
+import static com.google.gcloud.storage.StorageService.PredefinedAcl.PUBLIC_READ;
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Iterables;
 import com.google.gcloud.storage.StorageService.BlobSourceOption;
+import com.google.gcloud.storage.StorageService.BlobTargetOption;
 
 import org.junit.Test;
 
+import java.util.Iterator;
+import java.util.Map.Entry;
+
 public class BatchRequestTest {
 
   @Test
   public void testRequest() throws Exception {
-    Builder builder = BatchRequest.builder();
-    builder.delete("b1", "o1")
-    BatchRequest.builder().delete("b1", "o1",
-        BlobSourceOption.generationMatch(), BlobSourceOption.metagenerationMatch());
-  }
+    BatchRequest request = BatchRequest.builder()
+        .delete("b1", "o1")
+        .delete("b1", "o2", BlobSourceOption.generationMatch(1),
+            BlobSourceOption.metagenerationMatch(2))
+        .update(Blob.of("b2", "o1"), BlobTargetOption.predefinedAcl(PUBLIC_READ))
+        .update(Blob.of("b2", "o2"))
+        .get("b3", "o1")
+        .get("b3", "o2", BlobSourceOption.generationMatch(1))
+        .get("b3", "o3")
+        .build();
 
-  @Test
-  public void testToUpdate() throws Exception {
-
-  }
+    Iterator>> deletes = request
+        .toDelete().entrySet().iterator();
+    Entry> delete = deletes.next();
+    assertEquals(Blob.of("b1", "o1"), delete.getKey());
+    assertTrue(Iterables.isEmpty(delete.getValue()));
+    delete = deletes.next();
+    assertEquals(Blob.of("b1", "o2"), delete.getKey());
+    assertEquals(2, Iterables.size(delete.getValue()));
+    assertFalse(deletes.hasNext());
 
-  @Test
-  public void testToGet() throws Exception {
-
-  }
-
-  @Test
-  public void testBuilder() throws Exception {
+    Iterator>> updates = request
+        .toDelete().entrySet().iterator();
+    Entry> update = updates.next();
+    assertEquals(Blob.of("b1", "o1"), update.getKey());
+    assertTrue(Iterables.isEmpty(update.getValue()));
+    update = updates.next();
+    assertEquals(Blob.of("b1", "o2"), update.getKey());
+    assertEquals(2, Iterables.size(update.getValue()));
+    assertFalse(updates.hasNext());
 
   }
 }

From b1c06dd86288fe781ffa56fd9c5d9505ecd1c9c3 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 14 May 2015 20:30:19 -0700
Subject: [PATCH 224/732] basic tests

---
 .../google/gcloud/storage/BatchResponse.java  |  4 ++
 .../com/google/gcloud/storage/ListResult.java |  4 +-
 .../gcloud/storage/StorageServiceImpl.java    |  4 +-
 .../gcloud/storage/BatchRequestTest.java      | 32 ++++++++---
 .../gcloud/storage/BatchResponseTest.java     | 45 +++++++++++++++
 .../com/google/gcloud/storage/CorsTest.java   | 55 +++++++++++++++++++
 .../google/gcloud/storage/ListResultTest.java | 34 ++++++++++++
 .../com/google/gcloud/storage/OptionTest.java | 38 +++++++++++++
 .../gcloud/storage/SerializationTest.java     |  2 +-
 9 files changed, 206 insertions(+), 12 deletions(-)
 create mode 100644 src/test/java/com/google/gcloud/storage/BatchResponseTest.java
 create mode 100644 src/test/java/com/google/gcloud/storage/CorsTest.java
 create mode 100644 src/test/java/com/google/gcloud/storage/ListResultTest.java
 create mode 100644 src/test/java/com/google/gcloud/storage/OptionTest.java

diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index f0675e348f72..03e02ff01d3a 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -53,6 +53,10 @@ public static class Result implements Serializable {
       this.value = null;
     }
 
+    static  Result of(T value) {
+      return new Result<>(value);
+    }
+
     /**
      * Returns the result.
      *
diff --git a/src/main/java/com/google/gcloud/storage/ListResult.java b/src/main/java/com/google/gcloud/storage/ListResult.java
index dd843020376e..de7349328b9a 100644
--- a/src/main/java/com/google/gcloud/storage/ListResult.java
+++ b/src/main/java/com/google/gcloud/storage/ListResult.java
@@ -16,7 +16,9 @@
 
 package com.google.gcloud.storage;
 
+
 import java.io.Serializable;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Objects;
 
@@ -41,7 +43,7 @@ public String nextPageCursor() {
 
   @Override
   public Iterator iterator() {
-    return results.iterator();
+    return results == null ? Collections.emptyIterator() : results.iterator();
   }
 
   @Override
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 1f66401105d0..4507146a50f0 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -345,14 +345,14 @@ private  List> transformBatch
     for (Tuple tuple : request) {
       Tuple result = results.get(tuple.x());
       if (result.x() != null) {
-        response.add(new BatchResponse.Result<>(transform.apply(result.x())));
+        response.add(BatchResponse.Result.of(transform.apply(result.x())));
       } else {
         StorageServiceException exception = result.y();
         if (nullOnErrorCodesSet.contains(exception.code())) {
           //noinspection unchecked
           response.add(BatchResponse.Result.empty());
         } else {
-          response.add(new BatchResponse.Result(result.y()));
+          response.add(new BatchResponse.Result(exception));
         }
       }
     }
diff --git a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
index 56c1f72dd84d..445e65dc6c35 100644
--- a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
+++ b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
@@ -33,7 +33,7 @@
 public class BatchRequestTest {
 
   @Test
-  public void testRequest() throws Exception {
+  public void testBatchRequest() {
     BatchRequest request = BatchRequest.builder()
         .delete("b1", "o1")
         .delete("b1", "o2", BlobSourceOption.generationMatch(1),
@@ -55,15 +55,31 @@ public void testRequest() throws Exception {
     assertEquals(2, Iterables.size(delete.getValue()));
     assertFalse(deletes.hasNext());
 
-    Iterator>> updates = request
-        .toDelete().entrySet().iterator();
-    Entry> update = updates.next();
-    assertEquals(Blob.of("b1", "o1"), update.getKey());
-    assertTrue(Iterables.isEmpty(update.getValue()));
+    Iterator>> updates = request
+        .toUpdate().entrySet().iterator();
+    Entry> update = updates.next();
+    assertEquals(Blob.of("b2", "o1"), update.getKey());
+    assertEquals(1, Iterables.size(update.getValue()));
+    assertEquals(BlobTargetOption.predefinedAcl(PUBLIC_READ),
+        Iterables.getFirst(update.getValue(), null));
     update = updates.next();
-    assertEquals(Blob.of("b1", "o2"), update.getKey());
-    assertEquals(2, Iterables.size(update.getValue()));
+    assertEquals(Blob.of("b2", "o2"), update.getKey());
+    assertTrue(Iterables.isEmpty(update.getValue()));
     assertFalse(updates.hasNext());
 
+    Iterator>> gets = request
+        .toGet().entrySet().iterator();
+    Entry> get = gets.next();
+    assertEquals(Blob.of("b3", "o1"), get.getKey());
+    assertTrue(Iterables.isEmpty(get.getValue()));
+    get = gets.next();
+    assertEquals(Blob.of("b3", "o2"), get.getKey());
+    assertEquals(1, Iterables.size(get.getValue()));
+    assertEquals(BlobSourceOption.generationMatch(1),
+        Iterables.getFirst(get.getValue(), null));
+    get = gets.next();
+    assertEquals(Blob.of("b3", "o3"), get.getKey());
+    assertTrue(Iterables.isEmpty(get.getValue()));
+    assertFalse(gets.hasNext());
   }
 }
diff --git a/src/test/java/com/google/gcloud/storage/BatchResponseTest.java b/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
new file mode 100644
index 000000000000..8e6e850fca61
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.gcloud.storage.BatchResponse.Result;
+
+import org.junit.Test;
+
+import java.util.List;
+
+public class BatchResponseTest {
+
+  private static final Blob BLOB1 = Blob.of("b", "o1");
+  private static final Blob BLOB2 = Blob.of("b", "o2");
+  private static final Blob BLOB3 = Blob.of("b", "o3");
+
+  @Test
+  public void testBatchResponse() {
+    List> deletes = ImmutableList.of(Result.of(true), Result.of(false));
+    List> updates = ImmutableList.of(Result.of(BLOB1), Result.of(BLOB2));
+    List> gets = ImmutableList.of(Result.of(BLOB2), Result.of(BLOB3));
+    BatchResponse response = new BatchResponse(deletes, updates, gets);
+
+    assertEquals(deletes, response.deletes());
+    assertEquals(updates, response.updates());
+    assertEquals(gets, response.gets());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/storage/CorsTest.java b/src/test/java/com/google/gcloud/storage/CorsTest.java
new file mode 100644
index 000000000000..fcb343cd56f6
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/CorsTest.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.gcloud.storage.Cors.Method;
+import com.google.gcloud.storage.Cors.Origin;
+
+import org.junit.Test;
+
+import java.util.List;
+
+public class CorsTest {
+
+  @Test
+  public void testOrigin() {
+    assertEquals("bla", Origin.of("bla").value());
+    assertEquals("http://host:8080", Origin.of("http", "host", 8080).toString());
+    assertEquals(Origin.of("*"), Origin.any());
+  }
+
+  @Test
+  public void corsTest() {
+    List origins = ImmutableList.of(Origin.any(), Origin.of("o"));
+    List headers = ImmutableList.of("h1", "h2");
+    List methods = ImmutableList.of(Method.ANY);
+    Cors cors = Cors.builder()
+        .maxAgeSeconds(100)
+        .origins(origins)
+        .responseHeaders(headers)
+        .methods(methods)
+        .build();
+
+    assertEquals(Integer.valueOf(100), cors.maxAgeSeconds());
+    assertEquals(origins, cors.origins());
+    assertEquals(methods, cors.methods());
+    assertEquals(headers, cors.responseHeaders());
+  }
+}
diff --git a/src/test/java/com/google/gcloud/storage/ListResultTest.java b/src/test/java/com/google/gcloud/storage/ListResultTest.java
new file mode 100644
index 000000000000..3295e86d01ec
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/ListResultTest.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Test;
+
+public class ListResultTest {
+
+  @Test
+  public void testListResult() throws Exception {
+    ImmutableList values = ImmutableList.of("1", "2");
+    ListResult result = new ListResult("c", values);
+    assertEquals("c", result.nextPageCursor());
+    assertEquals(values, ImmutableList.copyOf(result.iterator()));
+  }
+}
diff --git a/src/test/java/com/google/gcloud/storage/OptionTest.java b/src/test/java/com/google/gcloud/storage/OptionTest.java
new file mode 100644
index 000000000000..b900ce2fc293
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/OptionTest.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static junit.framework.TestCase.assertEquals;
+
+import com.google.gcloud.spi.StorageRpc;
+
+import org.junit.Test;
+
+public class OptionTest {
+
+  @Test
+  public void testOption() {
+    Option option = new Option(StorageRpc.Option.DELIMITER, "/");
+    assertEquals(StorageRpc.Option.DELIMITER, option.rpcOption());
+    assertEquals("/", option.value());
+  }
+
+  @Test(expected=NullPointerException.class)
+  public void testIndexOutOfBoundsException() {
+    new Option(null, "/");
+  }
+}
diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/src/test/java/com/google/gcloud/storage/SerializationTest.java
index 365462d3f69a..97119698c1cf 100644
--- a/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -47,7 +47,7 @@ public class SerializationTest {
       Cors.builder().maxAgeSeconds(1).origins(Collections.singleton(ORIGIN)).build();
   private static final BatchRequest BATCH_REQUEST = BatchRequest.builder().delete("B", "N").build();
   private static final BatchResponse BATCH_RESPONSE = new BatchResponse(
-      Collections.singletonList(new BatchResponse.Result<>(true)),
+      Collections.singletonList(BatchResponse.Result.of(true)),
       Collections.>emptyList(),
       Collections.>emptyList());
   private static final ListResult LIST_RESULT =

From 808fcb8a49d8bb82abb31325b4fd865c4eb75ea2 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 15 May 2015 09:46:28 -0700
Subject: [PATCH 225/732] make Project constructor public

---
 src/main/java/com/google/gcloud/storage/Acl.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/src/main/java/com/google/gcloud/storage/Acl.java
index e2246bd29d75..d77bb1eaef02 100644
--- a/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/src/main/java/com/google/gcloud/storage/Acl.java
@@ -186,7 +186,7 @@ enum ProjectRole {
       OWNERS, EDITORS, VIEWERS
     }
 
-    Project(ProjectRole pRole, String projectId) {
+    public Project(ProjectRole pRole, String projectId) {
       super(Type.PROJECT, pRole.name().toLowerCase() + "-" + projectId);
       this.pRole = pRole;
       this.projectId = projectId;

From 2b3c93432870bbae89bd71a67a6753da45e7b322 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 15 May 2015 14:29:34 -0700
Subject: [PATCH 226/732] s/junit.framework.TestCase/org.junit.Assert/g

---
 .../google/gcloud/datastore/BaseEntityTest.java   |  2 +-
 .../google/gcloud/datastore/BlobValueTest.java    |  2 +-
 .../google/gcloud/datastore/BooleanValueTest.java |  2 +-
 .../gcloud/datastore/DatastoreHelperTest.java     | 15 +++++++++++----
 .../datastore/DatastoreServiceExceptionTest.java  |  2 +-
 .../datastore/DatastoreServiceOptionsTest.java    |  8 ++++----
 .../gcloud/datastore/DatastoreServiceTest.java    |  2 +-
 .../gcloud/datastore/DateTimeValueTest.java       |  2 +-
 .../google/gcloud/datastore/DoubleValueTest.java  |  2 +-
 .../google/gcloud/datastore/EntityValueTest.java  |  2 +-
 .../google/gcloud/datastore/KeyFactoryTest.java   |  4 ++--
 .../com/google/gcloud/datastore/KeyValueTest.java |  2 +-
 .../google/gcloud/datastore/ListValueTest.java    |  2 +-
 .../google/gcloud/datastore/LongValueTest.java    |  2 +-
 .../google/gcloud/datastore/NullValueTest.java    |  2 +-
 .../gcloud/datastore/ProjectionEntityTest.java    |  6 +++---
 .../com/google/gcloud/datastore/RawValueTest.java |  2 +-
 .../google/gcloud/datastore/StringValueTest.java  |  2 +-
 .../com/google/gcloud/datastore/ValueTest.java    |  4 ++--
 .../google/gcloud/storage/BatchRequestTest.java   |  2 +-
 .../google/gcloud/storage/BatchResponseTest.java  |  2 +-
 .../java/com/google/gcloud/storage/CorsTest.java  |  2 +-
 .../com/google/gcloud/storage/ListResultTest.java |  2 +-
 .../com/google/gcloud/storage/OptionTest.java     |  2 +-
 24 files changed, 41 insertions(+), 34 deletions(-)

diff --git a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
index 8a2ffb26d68e..567e795a66e7 100644
--- a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final FullEntity PARTIAL_ENTITY =
      Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override public BaseEntity build() {

      return new BaseEntity(this) {

        @Override
        protected Builder emptyBuilder() {
          return new BaseEntityTest.Builder();
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final FullEntity PARTIAL_ENTITY =
      Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override public BaseEntity build() {

      return new BaseEntity(this) {

        @Override
        protected Builder emptyBuilder() {
          return new BaseEntityTest.Builder();
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/BlobValueTest.java b/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
index 56ef03ac278e..40d0299d8fb3 100644
--- a/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java b/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
index fb3e62e6e27d..16bbe9cbf518 100644
--- a/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 25c0e8b2e691..8a31ece583ff 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -16,13 +16,20 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertTrue;
-import static junit.framework.TestCase.fail;
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.*;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import com.google.common.collect.Iterators;
 import com.google.gcloud.datastore.DatastoreService.TransactionCallable;
+
 import org.easymock.EasyMock;
 import org.junit.Test;
 
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
index 2700d277b9a3..1a06de833cf2 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;

import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
import com.google.gcloud.datastore.DatastoreServiceException.Code;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retryable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

import com.google.gcloud.datastore.DatastoreServiceException.Code;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retryable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
index a533f4ddc15e..59468be58129 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
@@ -16,10 +16,10 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertEquals;
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertNull;
-import static junit.framework.TestCase.assertSame;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 import com.google.gcloud.spi.DatastoreRpc;
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
index 0423cfe2d494..535fcb5f13e1 100644
--- a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
@@ -16,10 +16,10 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertNotNull;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
diff --git a/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java b/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
index 39e0d1ca3f01..d7fef2ca69b9 100644
--- a/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java b/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
index 012f9bbf4de9..fa39511a45de 100644
--- a/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
index dfacfb0b67a6..cd1f7af38067 100644
--- a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
index ca9ea08294f4..92851bd87efe 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertEquals;
-import static junit.framework.TestCase.assertNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Before;
diff --git a/src/test/java/com/google/gcloud/datastore/KeyValueTest.java b/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
index ea6b77ed97da..131a80462a62 100644
--- a/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/ListValueTest.java b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
index ae07f0c4fa51..04fdbec54727 100644
--- a/src/test/java/com/google/gcloud/datastore/ListValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ListValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import com.google.common.collect.ImmutableList;
diff --git a/src/test/java/com/google/gcloud/datastore/LongValueTest.java b/src/test/java/com/google/gcloud/datastore/LongValueTest.java
index d93dfaf331a8..c4c899785d68 100644
--- a/src/test/java/com/google/gcloud/datastore/LongValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/LongValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/NullValueTest.java b/src/test/java/com/google/gcloud/datastore/NullValueTest.java
index 15fe78dd1a4f..a42fdaf0229f 100644
--- a/src/test/java/com/google/gcloud/datastore/NullValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/NullValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
diff --git a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
index f67d719af265..0262fb04b89d 100644
--- a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
@@ -16,10 +16,10 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertEquals;
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertNull;
 import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/RawValueTest.java b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
index 26854ff9d22b..4d63bc89bacb 100644
--- a/src/test/java/com/google/gcloud/datastore/RawValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/RawValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import com.google.api.services.datastore.DatastoreV1;
diff --git a/src/test/java/com/google/gcloud/datastore/StringValueTest.java b/src/test/java/com/google/gcloud/datastore/StringValueTest.java
index aa53cea22d3a..a2cacd6574aa 100644
--- a/src/test/java/com/google/gcloud/datastore/StringValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/StringValueTest.java
@@ -16,8 +16,8 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
diff --git a/src/test/java/com/google/gcloud/datastore/ValueTest.java b/src/test/java/com/google/gcloud/datastore/ValueTest.java
index 51304e9f1cac..bbfb790b69a2 100644
--- a/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -16,10 +16,10 @@
 
 package com.google.gcloud.datastore;
 
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertTrue;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
diff --git a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
index 445e65dc6c35..dbe0eaa19411 100644
--- a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
+++ b/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
@@ -17,8 +17,8 @@
 package com.google.gcloud.storage;
 
 import static com.google.gcloud.storage.StorageService.PredefinedAcl.PUBLIC_READ;
-import static junit.framework.TestCase.assertFalse;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import com.google.common.collect.Iterables;
diff --git a/src/test/java/com/google/gcloud/storage/BatchResponseTest.java b/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
index 8e6e850fca61..277c46860ef1 100644
--- a/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
+++ b/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
@@ -16,7 +16,7 @@
 
 package com.google.gcloud.storage;
 
-import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertEquals;
 
 import com.google.common.collect.ImmutableList;
 import com.google.gcloud.storage.BatchResponse.Result;
diff --git a/src/test/java/com/google/gcloud/storage/CorsTest.java b/src/test/java/com/google/gcloud/storage/CorsTest.java
index fcb343cd56f6..8b0379f03583 100644
--- a/src/test/java/com/google/gcloud/storage/CorsTest.java
+++ b/src/test/java/com/google/gcloud/storage/CorsTest.java
@@ -16,7 +16,7 @@
 
 package com.google.gcloud.storage;
 
-import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertEquals;
 
 import com.google.common.collect.ImmutableList;
 import com.google.gcloud.storage.Cors.Method;
diff --git a/src/test/java/com/google/gcloud/storage/ListResultTest.java b/src/test/java/com/google/gcloud/storage/ListResultTest.java
index 3295e86d01ec..1345b0ced240 100644
--- a/src/test/java/com/google/gcloud/storage/ListResultTest.java
+++ b/src/test/java/com/google/gcloud/storage/ListResultTest.java
@@ -16,7 +16,7 @@
 
 package com.google.gcloud.storage;
 
-import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertEquals;
 
 import com.google.common.collect.ImmutableList;
 
diff --git a/src/test/java/com/google/gcloud/storage/OptionTest.java b/src/test/java/com/google/gcloud/storage/OptionTest.java
index b900ce2fc293..4665d04b2d82 100644
--- a/src/test/java/com/google/gcloud/storage/OptionTest.java
+++ b/src/test/java/com/google/gcloud/storage/OptionTest.java
@@ -16,7 +16,7 @@
 
 package com.google.gcloud.storage;
 
-import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertEquals;
 
 import com.google.gcloud.spi.StorageRpc;
 

From 9b54b495a45c620b1a3f364c3a0dc031f1052cea Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 15 May 2015 15:27:44 -0700
Subject: [PATCH 227/732] add tests for bucket and blob

---
 .../com/google/gcloud/storage/Bucket.java     |  43 ++++-
 .../java/com/google/gcloud/storage/Cors.java  |  42 +++--
 .../com/google/gcloud/storage/BlobTest.java   | 155 ++++++++++++++++
 .../com/google/gcloud/storage/BucketTest.java | 175 ++++++++++++++++++
 4 files changed, 394 insertions(+), 21 deletions(-)
 create mode 100644 src/test/java/com/google/gcloud/storage/BlobTest.java
 create mode 100644 src/test/java/com/google/gcloud/storage/BucketTest.java

diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/src/main/java/com/google/gcloud/storage/Bucket.java
index 793ae0199d5c..85d601487c28 100644
--- a/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/src/main/java/com/google/gcloud/storage/Bucket.java
@@ -103,6 +103,23 @@ public Type type() {
       return type;
     }
 
+    @Override
+    public int hashCode() {
+      return Objects.hash(type);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null || getClass() != obj.getClass()) {
+        return false;
+      }
+      final DeleteRule other = (DeleteRule) obj;
+      return Objects.equals(toPb(), other.toPb());
+    }
+
     Rule toPb() {
       Rule rule = new Rule();
       rule.setAction(new Rule.Action().setType(SUPPORTED_ACTION));
@@ -114,7 +131,7 @@ Rule toPb() {
 
     abstract void populateCondition(Rule.Condition condition);
 
-    private static DeleteRule fromPb(Rule rule) {
+    static DeleteRule fromPb(Rule rule) {
       if (rule.getAction() != null && SUPPORTED_ACTION.endsWith(rule.getAction().getType())) {
         Rule.Condition condition = rule.getCondition();
         Integer age = condition.getAge();
@@ -346,6 +363,23 @@ public static Location of(String value) {
       return option == null ? new Location(value) : option.location;
     }
 
+    @Override
+    public int hashCode() {
+      return Objects.hash(value);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null || getClass() != obj.getClass()) {
+        return false;
+      }
+      final Location other = (Location) obj;
+      return Objects.equals(this.value, other.value);
+    }
+
     @Override
     public String toString() {
       return value();
@@ -412,7 +446,7 @@ public Builder notFoundPage(String notFoundPage) {
       return this;
     }
 
-    public Builder deleteRules(Iterable rules) {
+    public Builder deleteRules(Iterable rules) {
       this.deleteRules = ImmutableList.copyOf(rules);
       return this;
     }
@@ -490,7 +524,7 @@ public String name() {
     return name;
   }
 
-  public Entity Owner() {
+  public Entity owner() {
     return owner;
   }
 
@@ -510,7 +544,7 @@ public String notFoundPage() {
     return notFoundPage;
   }
 
-  public List deleteRules() {
+  public List deleteRules() {
     return deleteRules;
   }
 
@@ -652,6 +686,7 @@ public Rule apply(DeleteRule deleteRule) {
           return deleteRule.toPb();
         }
       }));
+      bucketPb.setLifecycle(lifecycle);
     }
     return bucketPb;
   }
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index 9cedbc0d3b4c..b1953aa5e0e4 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -130,17 +130,17 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) {
     }
 
     public Builder methods(Iterable methods) {
-      this.methods = ImmutableList.copyOf(methods);
+      this.methods = methods != null ? ImmutableList.copyOf(methods) : null;
       return this;
     }
 
     public Builder origins(Iterable origins) {
-      this.origins = ImmutableList.copyOf(origins);
+      this.origins = origins != null ? ImmutableList.copyOf(origins) : null;
       return this;
     }
 
     public Builder responseHeaders(Iterable headers) {
-      this.responseHeaders = ImmutableList.copyOf(headers);
+      this.responseHeaders = headers != null ? ImmutableList.copyOf(headers) : null;
       return this;
     }
 
@@ -205,25 +205,33 @@ Bucket.Cors toPb() {
     Bucket.Cors pb = new Bucket.Cors();
     pb.setMaxAgeSeconds(maxAgeSeconds);
     pb.setResponseHeader(responseHeaders);
-    pb.setMethod(newArrayList(transform(methods(), Functions.toStringFunction())));
-    pb.setOrigin(newArrayList(transform(origins(), Functions.toStringFunction())));
+    if (methods != null) {
+      pb.setMethod(newArrayList(transform(methods, Functions.toStringFunction())));
+    }
+    if (origins != null) {
+      pb.setOrigin(newArrayList(transform(origins, Functions.toStringFunction())));
+    }
     return pb;
   }
 
   static Cors fromPb(Bucket.Cors cors) {
     Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds());
-    builder.methods(transform(cors.getMethod(), new Function() {
-      @Override
-      public Method apply(String name) {
-        return Method.valueOf(name.toUpperCase());
-      }
-    }));
-    builder.origins(transform(cors.getOrigin(), new Function() {
-      @Override
-      public Origin apply(String value) {
-        return Origin.of(value);
-      }
-    }));
+    if (cors.getMethod() != null) {
+      builder.methods(transform(cors.getMethod(), new Function() {
+        @Override
+        public Method apply(String name) {
+          return Method.valueOf(name.toUpperCase());
+        }
+      }));
+    }
+    if (cors.getOrigin() != null) {
+      builder.origins(transform(cors.getOrigin(), new Function() {
+        @Override
+        public Origin apply(String value) {
+          return Origin.of(value);
+        }
+      }));
+    }
     builder.responseHeaders(cors.getResponseHeader());
     return builder.build();
   }
diff --git a/src/test/java/com/google/gcloud/storage/BlobTest.java b/src/test/java/com/google/gcloud/storage/BlobTest.java
new file mode 100644
index 000000000000..6fb8f0243987
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/BlobTest.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static com.google.gcloud.storage.Acl.Project.ProjectRole.VIEWERS;
+import static com.google.gcloud.storage.Acl.Role.READER;
+import static com.google.gcloud.storage.Acl.Role.WRITER;
+import static org.junit.Assert.assertEquals;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.gcloud.storage.Acl.Project;
+import com.google.gcloud.storage.Acl.User;
+
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+public class BlobTest {
+
+  private static final List ACL = ImmutableList.of(
+      new Acl(User.ofAllAuthenticatedUsers(), READER),
+      new Acl(new Project(VIEWERS, "p1"), WRITER));
+  private static final Integer COMPONENT_COUNT = 2;
+  private static final String CONTENT_TYPE = "text/html";
+  private static final String CACHE_CONTROL = "cache";
+  private static final String CONTENT_DISPOSITION = "content-disposition";
+  private static final String CONTENT_ENCODING = "UTF-8";
+  private static final String CONTENT_LANGUAGE = "En";
+  private static final String CRC32 = "0xFF00";
+  private static final Long DELETE_TIME = System.currentTimeMillis();
+  private static final String ETAG = "0xFF00";
+  private static final Long GENERATION = 1L;
+  private static final String ID = "B/N:1";
+  private static final String MD5 = "0xFF00";
+  private static final String MEDIA_LINK = "http://media/b/n";
+  private static final Map METADATA = ImmutableMap.of("n1", "v1", "n2", "v2");
+  private static final Long META_GENERATION = 10L;
+  private static final User OWNER = new User("user@gmail.com");
+  private static final String SELF_LINK = "http://storage/b/n";
+  private static final Long SIZE = 1024L;
+  private static final Long UPDATE_TIME = DELETE_TIME - 1L;
+  private static final Blob BLOB = Blob.builder("b", "n")
+      .acl(ACL)
+      .componentCount(COMPONENT_COUNT)
+      .contentType(CONTENT_TYPE)
+      .cacheControl(CACHE_CONTROL)
+      .contentDisposition(CONTENT_DISPOSITION)
+      .contentEncoding(CONTENT_ENCODING)
+      .contentLanguage(CONTENT_LANGUAGE)
+      .crc32c(CRC32)
+      .deleteTime(DELETE_TIME)
+      .etag(ETAG)
+      .generation(GENERATION)
+      .id(ID)
+      .md5(MD5)
+      .mediaLink(MEDIA_LINK)
+      .metadata(METADATA)
+      .metageneration(META_GENERATION)
+      .owner(OWNER)
+      .selfLink(SELF_LINK)
+      .size(SIZE)
+      .updateTime(UPDATE_TIME)
+      .build();
+
+  @Test
+  public void testToBuilder() {
+    compareBlobs(BLOB, BLOB.toBuilder().build());
+    Blob blob = BLOB.toBuilder().name("n2").bucket("b2").size(200L).build();
+    assertEquals("n2", blob.name());
+    assertEquals("b2", blob.bucket());
+    assertEquals(Long.valueOf(200), blob.size());
+    blob = blob.toBuilder().name("n").bucket("b").size(SIZE).build();
+    compareBlobs(BLOB, blob);
+  }
+
+  @Test
+  public void testOf() {
+    Blob blob = Blob.of("b", "n");
+    assertEquals("b", blob.bucket());
+    assertEquals("n", blob.name());
+  }
+
+  @Test
+  public void testBuilder() {
+    assertEquals("b", BLOB.bucket());
+    assertEquals("n", BLOB.name());
+    assertEquals(ACL, BLOB.acl());
+    assertEquals(COMPONENT_COUNT, BLOB.componentCount());
+    assertEquals(CONTENT_TYPE, BLOB.contentType());
+    assertEquals(CACHE_CONTROL, BLOB.cacheControl() );
+    assertEquals(CONTENT_DISPOSITION, BLOB.contentDisposition());
+    assertEquals(CONTENT_ENCODING, BLOB.contentEncoding());
+    assertEquals(CONTENT_LANGUAGE, BLOB.contentLanguage());
+    assertEquals(CRC32, BLOB.crc32c());
+    assertEquals(DELETE_TIME, BLOB.deleteTime());
+    assertEquals(ETAG, BLOB.etag());
+    assertEquals(GENERATION, BLOB.generation());
+    assertEquals(ID, BLOB.id());
+    assertEquals(MD5, BLOB.md5());
+    assertEquals(MEDIA_LINK, BLOB.mediaLink());
+    assertEquals(METADATA, BLOB.metadata());
+    assertEquals(META_GENERATION, BLOB.metageneration());
+    assertEquals(OWNER, BLOB.owner());
+    assertEquals(SELF_LINK, BLOB.selfLink());
+    assertEquals(SIZE, BLOB.size());
+    assertEquals(UPDATE_TIME, BLOB.updateTime());
+  }
+
+  private void compareBlobs(Blob expected, Blob value) {
+    assertEquals(expected, value);
+    assertEquals(expected.bucket(), value.bucket());
+    assertEquals(expected.name(), value.name());
+    assertEquals(expected.acl(), value.acl());
+    assertEquals(expected.componentCount(), value.componentCount());
+    assertEquals(expected.contentType(), value.contentType());
+    assertEquals(expected.cacheControl(), value.cacheControl() );
+    assertEquals(expected.contentDisposition(), value.contentDisposition());
+    assertEquals(expected.contentEncoding(), value.contentEncoding());
+    assertEquals(expected.contentLanguage(), value.contentLanguage());
+    assertEquals(expected.crc32c(), value.crc32c());
+    assertEquals(expected.deleteTime(), value.deleteTime());
+    assertEquals(expected.etag(), value.etag());
+    assertEquals(expected.generation(), value.generation());
+    assertEquals(expected.id(), value.id());
+    assertEquals(expected.md5(), value.md5());
+    assertEquals(expected.mediaLink(), value.mediaLink());
+    assertEquals(expected.metadata(), value.metadata());
+    assertEquals(expected.metageneration(), value.metageneration());
+    assertEquals(expected.owner(), value.owner());
+    assertEquals(expected.selfLink(), value.selfLink());
+    assertEquals(expected.size(), value.size());
+    assertEquals(expected.updateTime(), value.updateTime());
+  }
+
+  @Test
+  public void testToPbAndFromPb() {
+    compareBlobs(BLOB, Blob.fromPb(BLOB.toPb()));
+  }
+}
diff --git a/src/test/java/com/google/gcloud/storage/BucketTest.java b/src/test/java/com/google/gcloud/storage/BucketTest.java
new file mode 100644
index 000000000000..c095a6291e49
--- /dev/null
+++ b/src/test/java/com/google/gcloud/storage/BucketTest.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static com.google.gcloud.storage.Acl.Project.ProjectRole.VIEWERS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+import com.google.api.services.storage.model.Bucket.Lifecycle.Rule;
+import com.google.common.collect.ImmutableList;
+import com.google.gcloud.storage.Acl.Project;
+import com.google.gcloud.storage.Acl.Role;
+import com.google.gcloud.storage.Acl.User;
+import com.google.gcloud.storage.Bucket.AgeDeleteRule;
+import com.google.gcloud.storage.Bucket.CreatedBeforeDeleteRule;
+import com.google.gcloud.storage.Bucket.DeleteRule;
+import com.google.gcloud.storage.Bucket.DeleteRule.Type;
+import com.google.gcloud.storage.Bucket.IsLiveDeleteRule;
+import com.google.gcloud.storage.Bucket.Location;
+import com.google.gcloud.storage.Bucket.NumNewerVersionsDeleteRule;
+import com.google.gcloud.storage.Bucket.RawDeleteRule;
+import com.google.gcloud.storage.Bucket.StorageClass;
+
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+public class BucketTest {
+
+  private static final List ACL = ImmutableList.of(
+      new Acl(User.ofAllAuthenticatedUsers(), Role.READER),
+      new Acl(new Project(VIEWERS, "p1"), Role.WRITER));
+  private static final String ETAG = "0xFF00";
+  private static final String ID = "B/N:1";
+  private static final Long META_GENERATION = 10L;
+  private static final User OWNER = new User("user@gmail.com");
+  private static final String SELF_LINK = "http://storage/b/n";
+  private static final Long CREATE_TIME = System.currentTimeMillis();
+  private static final List CORS = Collections.singletonList(Cors.builder().build());
+  private static final List DEFAULT_ACL =
+      Collections.singletonList(new Acl(User.ofAllAuthenticatedUsers(), Role.WRITER));
+  private static final List DELETE_RULES =
+      Collections.singletonList(new AgeDeleteRule(5));
+  private static final String INDEX_PAGE = "index.html";
+  private static final String NOT_FOUND_PAGE = "error.html";
+  private static final Location LOCATION = Location.asia();
+  private static final StorageClass STORAGE_CLASS = StorageClass.standard();
+  private static final Boolean VERSIONING_ENABLED = true;
+  private static final Bucket BUCKET = Bucket.builder("b")
+      .acl(ACL)
+      .etag(ETAG)
+      .id(ID)
+      .metageneration(META_GENERATION)
+      .owner(OWNER)
+      .selfLink(SELF_LINK)
+      .cors(CORS)
+      .createTime(CREATE_TIME)
+      .defaultAcl(DEFAULT_ACL)
+      .deleteRules(DELETE_RULES)
+      .indexPage(INDEX_PAGE)
+      .notFoundPage(NOT_FOUND_PAGE)
+      .location(LOCATION)
+      .storageClass(STORAGE_CLASS)
+      .versioningEnabled(VERSIONING_ENABLED)
+      .build();
+
+  @Test
+  public void testToBuilder() {
+    compareBuckets(BUCKET, BUCKET.toBuilder().build());
+    Bucket bucket = BUCKET.toBuilder().name("B").id("id").build();
+    assertEquals("B", bucket.name());
+    assertEquals("id", bucket.id());
+    bucket = bucket.toBuilder().name("b").id(ID).build();
+    compareBuckets(BUCKET, bucket);
+  }
+
+  @Test
+  public void testOf() {
+    Bucket bucket = Bucket.of("bucket");
+    assertEquals("bucket", bucket.name());
+  }
+
+  @Test
+  public void testBuilder() {
+    assertEquals("b", BUCKET.name());
+    assertEquals(ACL, BUCKET.acl());
+    assertEquals(ETAG, BUCKET.etag());
+    assertEquals(ID, BUCKET.id());
+    assertEquals(META_GENERATION, BUCKET.metageneration());
+    assertEquals(OWNER, BUCKET.owner());
+    assertEquals(SELF_LINK, BUCKET.selfLink());
+    assertEquals(CREATE_TIME, BUCKET.createTime());
+    assertEquals(CORS, BUCKET.cors());
+    assertEquals(DEFAULT_ACL, BUCKET.defaultAcl());
+    assertEquals(DELETE_RULES, BUCKET.deleteRules());
+    assertEquals(INDEX_PAGE, BUCKET.indexPage());
+    assertEquals(NOT_FOUND_PAGE, BUCKET.notFoundPage());
+    assertEquals(LOCATION, BUCKET.location());
+    assertEquals(STORAGE_CLASS, BUCKET.storageClass());
+    assertEquals(VERSIONING_ENABLED, BUCKET.versioningEnabled());
+  }
+
+  @Test
+  public void testToPbAndFromPb() {
+    compareBuckets(BUCKET, Bucket.fromPb(BUCKET.toPb()));
+  }
+
+  private void compareBuckets(Bucket expected, Bucket value) {
+    assertEquals(expected, value);
+    assertEquals(expected.name(), value.name());
+    assertEquals(expected.acl(), value.acl());
+    assertEquals(expected.etag(), value.etag());
+    assertEquals(expected.id(), value.id());
+    assertEquals(expected.metageneration(), value.metageneration());
+    assertEquals(expected.owner(), value.owner());
+    assertEquals(expected.selfLink(), value.selfLink());
+    assertEquals(expected.createTime(), value.createTime());
+    assertEquals(expected.cors(), value.cors());
+    assertEquals(expected.defaultAcl(), value.defaultAcl());
+    assertEquals(expected.deleteRules(), value.deleteRules());
+    assertEquals(expected.indexPage(), value.indexPage());
+    assertEquals(expected.notFoundPage(), value.notFoundPage());
+    assertEquals(expected.location(), value.location());
+    assertEquals(expected.storageClass(), value.storageClass());
+    assertEquals(expected.versioningEnabled(), value.versioningEnabled());
+  }
+
+  public void testLocation() {
+    assertEquals("ASIA", Location.asia().value());
+    assertEquals("EN", Location.eu().value());
+    assertEquals("US", Location.us().value());
+    assertSame(Location.asia(), Location.of("asia"));
+    assertSame(Location.asia(), Location.of("EU"));
+    assertSame(Location.asia(), Location.of("uS"));
+  }
+
+  public void testDeleteRules() {
+    AgeDeleteRule ageRule = new AgeDeleteRule(10);
+    assertEquals(10, ageRule.daysToLive());
+    assertEquals(Type.AGE, ageRule.type());
+    CreatedBeforeDeleteRule createBeforeRule = new CreatedBeforeDeleteRule(1);
+    assertEquals(10, createBeforeRule.timeMillis());
+    assertEquals(Type.CREATE_BEFORE, createBeforeRule.type());
+    NumNewerVersionsDeleteRule versionsRule = new NumNewerVersionsDeleteRule(2);
+    assertEquals(2, versionsRule.numNewerVersions());
+    assertEquals(Type.NUM_NEWER_VERSIONS, versionsRule.type());
+    IsLiveDeleteRule isLiveRule = new IsLiveDeleteRule(true);
+    assertTrue(isLiveRule.isLive());
+    assertEquals(Type.IS_LIVE, isLiveRule.type());
+    Rule rule = new Rule().set("a", "b");
+    RawDeleteRule rawRule = new RawDeleteRule(rule);
+    assertEquals(Type.UNKNOWN, isLiveRule.type());
+    ImmutableList rules = ImmutableList
+        .of(ageRule, createBeforeRule, versionsRule, isLiveRule, rawRule);
+    for (DeleteRule delRule : rules) {
+      assertEquals(delRule, DeleteRule.fromPb(delRule.toPb()));
+    }
+  }
+}

From 53170fb9e5835716a4bf763823fc0a6f86b4453a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 15 May 2015 17:42:04 -0700
Subject: [PATCH 228/732] adding an option to page from ListResult

---
 .../com/google/gcloud/ServiceOptions.java     |   2 +-
 .../datastore/DatastoreServiceOptions.java    |  10 +-
 .../google/gcloud/storage/BatchRequest.java   |   6 +-
 .../google/gcloud/storage/BatchResponse.java  |   6 +-
 .../com/google/gcloud/storage/ListResult.java |  15 +-
 .../gcloud/storage/StorageServiceImpl.java    | 143 +++++++++++++-----
 .../gcloud/storage/StorageServiceOptions.java |  10 +-
 7 files changed, 139 insertions(+), 53 deletions(-)

diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/src/main/java/com/google/gcloud/ServiceOptions.java
index a72cb74185c7..448b617372b0 100644
--- a/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -283,7 +283,7 @@ public AuthCredentials authCredentials() {
   }
 
   public RetryParams retryParams() {
-    return retryParams;
+    return retryParams != null ? retryParams : RetryParams.noRetries();
   }
 
   public ServiceRpcFactory serviceRpcFactory() {
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 6bed641c7636..f7b1965d0b29 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -44,6 +44,7 @@ public class DatastoreServiceOptions extends ServiceOptions {
@@ -178,10 +179,15 @@ public boolean equals(Object obj) {
   }
 
   DatastoreRpc datastoreRpc() {
+    if (datastoreRpc != null) {
+      return datastoreRpc;
+    }
     if (serviceRpcFactory() != null) {
-      return serviceRpcFactory().create(this);
+      datastoreRpc = serviceRpcFactory().create(this);
+    } else {
+      datastoreRpc = ServiceRpcProvider.datastore(this);
     }
-    return ServiceRpcProvider.datastore(this);
+    return datastoreRpc;
   }
 
   public static DatastoreServiceOptions defaultInstance() {
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 6f62d5c51ae4..4d2f0cab8c96 100644
--- a/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -96,15 +96,15 @@ public boolean equals(Object obj) {
         && Objects.equals(toGet, other.toGet);
   }
 
-  Map> toDelete() {
+  public Map> toDelete() {
     return toDelete;
   }
 
-  Map> toUpdate() {
+  public Map> toUpdate() {
     return toUpdate;
   }
 
-  Map> toGet() {
+  public Map> toGet() {
     return toGet;
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/src/main/java/com/google/gcloud/storage/BatchResponse.java
index f0675e348f72..d4a1cc6f812f 100644
--- a/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -43,12 +43,12 @@ public static class Result implements Serializable {
     private final StorageServiceException exception;
 
 
-    Result(T value) {
+    public Result(T value) {
       this.value = value;
       this.exception = null;
     }
 
-    Result(StorageServiceException exception) {
+    public Result(StorageServiceException exception) {
       this.exception = exception;
       this.value = null;
     }
@@ -108,7 +108,7 @@ static  Result empty() {
     }
   }
 
-  BatchResponse(List> deleteResult, List> updateResult,
+  public BatchResponse(List> deleteResult, List> updateResult,
       List> getResult) {
     this.deleteResult = ImmutableList.copyOf(deleteResult);
     this.updateResult = ImmutableList.copyOf(updateResult);
diff --git a/src/main/java/com/google/gcloud/storage/ListResult.java b/src/main/java/com/google/gcloud/storage/ListResult.java
index dd843020376e..55406f90cf0c 100644
--- a/src/main/java/com/google/gcloud/storage/ListResult.java
+++ b/src/main/java/com/google/gcloud/storage/ListResult.java
@@ -29,8 +29,14 @@ public final class ListResult implements Iterable, Se
 
   private final String cursor;
   private final Iterable results;
+  private final NextPageFetcher pageFetcher;
 
-  ListResult(String cursor, Iterable results) {
+  interface NextPageFetcher extends Serializable {
+    ListResult nextPage();
+  }
+
+  public ListResult(NextPageFetcher pageFetcher, String cursor, Iterable results) {
+    this.pageFetcher = pageFetcher;
     this.cursor = cursor;
     this.results = results;
   }
@@ -39,6 +45,13 @@ public String nextPageCursor() {
     return cursor;
   }
 
+  public ListResult nextPage() {
+    if (cursor == null || pageFetcher == null) {
+      return null;
+    }
+    return pageFetcher.nextPage();
+  }
+
   @Override
   public Iterator iterator() {
     return results.iterator();
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 1f66401105d0..e0e77b0ec7e5 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -43,7 +43,6 @@
 import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
 import com.google.gcloud.ExceptionHandler.Interceptor;
-import com.google.gcloud.RetryParams;
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
@@ -83,12 +82,10 @@ public RetryResult beforeEval(Exception exception) {
   private static final byte[] EMPTY_BYTE_ARRAY = {};
 
   private final StorageRpc storageRpc;
-  private final RetryParams retryParams;
 
   StorageServiceImpl(StorageServiceOptions options) {
     super(options);
     storageRpc = options.storageRpc();
-    retryParams = firstNonNull(options.retryParams(), RetryParams.noRetries());
     // todo: replace nulls with Value.asNull (per toPb)
     // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors
     // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite
@@ -106,7 +103,7 @@ public Bucket create(Bucket bucket, BucketTargetOption... options) {
           public com.google.api.services.storage.model.Bucket call() {
             return storageRpc.create(bucketPb, optionsMap);
           }
-        }, retryParams, EXCEPTION_HANDLER));
+        }, options().retryParams(), EXCEPTION_HANDLER));
   }
 
   @Override
@@ -118,7 +115,7 @@ public Blob create(Blob blob, final byte[] content, BlobTargetOption... options)
       public StorageObject call() {
         return storageRpc.create(blobPb, firstNonNull(content, EMPTY_BYTE_ARRAY), optionsMap);
       }
-    }, retryParams, EXCEPTION_HANDLER));
+    }, options().retryParams(), EXCEPTION_HANDLER));
   }
 
   @Override
@@ -138,7 +135,7 @@ public com.google.api.services.storage.model.Bucket call() {
               throw ex;
             }
           }
-        }, retryParams, EXCEPTION_HANDLER);
+        }, options().retryParams(), EXCEPTION_HANDLER);
     return answer == null ? null : Bucket.fromPb(answer);
   }
 
@@ -158,46 +155,111 @@ public StorageObject call() {
           throw ex;
         }
       }
-    }, retryParams, EXCEPTION_HANDLER);
+    }, options().retryParams(), EXCEPTION_HANDLER);
     return storageObject == null ? null : Blob.fromPb(storageObject);
   }
 
+  private static abstract class BasePageFetcher
+      implements ListResult.NextPageFetcher {
+
+    private static final long serialVersionUID = 8236329004030295223L;
+    protected final Map requestOptions;
+    protected final StorageServiceOptions serviceOptions;
+
+    BasePageFetcher(StorageServiceOptions serviceOptions, String cursor,
+        Map optionMap) {
+      this.serviceOptions = serviceOptions;
+      ImmutableMap.Builder builder = ImmutableMap.builder();
+      builder.put(StorageRpc.Option.PAGE_TOKEN, cursor);
+      for (Map.Entry option : optionMap.entrySet()) {
+        if (option.getKey() != StorageRpc.Option.PAGE_TOKEN) {
+          builder.put(option.getKey(), option.getValue());
+        }
+      }
+      this.requestOptions = builder.build();
+    }
+  }
+
+  private static class BucketPageFetcher extends BasePageFetcher {
+
+    private static final long serialVersionUID = -5490616010200159174L;
+
+    BucketPageFetcher(StorageServiceOptions serviceOptions, String cursor,
+        Map optionMap) {
+      super(serviceOptions, cursor, optionMap);
+    }
+
+    @Override
+    public ListResult nextPage() {
+      return listBuckets(serviceOptions, requestOptions);
+    }
+  }
+
+  private static class BlobPageFetcher extends BasePageFetcher {
+
+    private static final long serialVersionUID = -5490616010200159174L;
+    private final String bucket;
+
+    BlobPageFetcher(String bucket, StorageServiceOptions serviceOptions, String cursor,
+        Map optionMap) {
+      super(serviceOptions, cursor, optionMap);
+      this.bucket = bucket;
+    }
+
+    @Override
+    public ListResult nextPage() {
+      return listBlobs(bucket, serviceOptions, requestOptions);
+    }
+  }
+
   @Override
   public ListResult list(BucketListOption... options) {
-    final Map optionsMap = optionMap(options);
+    return listBuckets(options(), optionMap(options));
+  }
+
+  private static ListResult listBuckets(final StorageServiceOptions serviceOptions,
+      final Map optionsMap) {
     Tuple> result = runWithRetries(
         new Callable>>() {
           @Override
           public Tuple> call() {
-            return storageRpc.list(optionsMap);
-          }
-        }, retryParams, EXCEPTION_HANDLER);
-    return new ListResult<>(result.x(), Iterables.transform(result.y(),
-        new Function() {
-          @Override
-          public Bucket apply(com.google.api.services.storage.model.Bucket bucketPb) {
-            return Bucket.fromPb(bucketPb);
+            return serviceOptions.storageRpc().list(optionsMap);
           }
-        }));
+        }, serviceOptions.retryParams(), EXCEPTION_HANDLER);
+    String cursor = result.x();
+    return new ListResult<>(new BucketPageFetcher(serviceOptions, cursor, optionsMap), cursor,
+        Iterables.transform(result.y(),
+            new Function() {
+              @Override
+              public Bucket apply(com.google.api.services.storage.model.Bucket bucketPb) {
+                return Bucket.fromPb(bucketPb);
+              }
+            }));
   }
 
   @Override
   public ListResult list(final String bucket, BlobListOption... options) {
-    final Map optionsMap = optionMap(options);
+    return listBlobs(bucket, options(), optionMap(options));
+  }
+
+  private static ListResult listBlobs(final String bucket,
+      final StorageServiceOptions serviceOptions, final Map optionsMap) {
     Tuple> result = runWithRetries(
         new Callable>>() {
           @Override
           public Tuple> call() {
-            return storageRpc.list(bucket, optionsMap);
+            return serviceOptions.storageRpc().list(bucket, optionsMap);
           }
-        }, retryParams, EXCEPTION_HANDLER);
-    return new ListResult<>(result.x(), Iterables.transform(result.y(),
-        new Function() {
-          @Override
-          public Blob apply(StorageObject storageObject) {
-            return Blob.fromPb(storageObject);
-          }
-        }));
+        }, serviceOptions.retryParams(), EXCEPTION_HANDLER);
+    String cursor = result.x();
+    return new ListResult<>(new BlobPageFetcher(bucket, serviceOptions, cursor, optionsMap), cursor,
+        Iterables.transform(result.y(),
+            new Function() {
+              @Override
+              public Blob apply(StorageObject storageObject) {
+                return Blob.fromPb(storageObject);
+              }
+            }));
   }
 
   @Override
@@ -210,7 +272,7 @@ public Bucket update(Bucket bucket, BucketTargetOption... options) {
           public com.google.api.services.storage.model.Bucket call() {
             return storageRpc.patch(bucketPb, optionsMap);
           }
-        }, retryParams, EXCEPTION_HANDLER));
+        }, options().retryParams(), EXCEPTION_HANDLER));
   }
 
   @Override
@@ -222,7 +284,7 @@ public Blob update(Blob blob, BlobTargetOption... options) {
       public StorageObject call() {
         return storageRpc.patch(storageObject, optionsMap);
       }
-    }, retryParams, EXCEPTION_HANDLER));
+    }, options().retryParams(), EXCEPTION_HANDLER));
   }
 
   @Override
@@ -234,7 +296,7 @@ public boolean delete(String bucket, BucketSourceOption... options) {
       public Boolean call() {
         return storageRpc.delete(bucketPb, optionsMap);
       }
-    }, retryParams, EXCEPTION_HANDLER);
+    }, options().retryParams(), EXCEPTION_HANDLER);
   }
 
   @Override
@@ -246,7 +308,7 @@ public boolean delete(String bucket, String blob, BlobSourceOption... options) {
       public Boolean call() {
         return storageRpc.delete(storageObject, optionsMap);
       }
-    }, retryParams, EXCEPTION_HANDLER);
+    }, options().retryParams(), EXCEPTION_HANDLER);
   }
 
   @Override
@@ -265,7 +327,7 @@ public Blob compose(final ComposeRequest composeRequest) {
       public StorageObject call() {
         return storageRpc.compose(sources, target, targetOptions);
       }
-    }, retryParams, EXCEPTION_HANDLER));
+    }, options().retryParams(), EXCEPTION_HANDLER));
   }
 
   @Override
@@ -283,7 +345,7 @@ public Blob copy(CopyRequest copyRequest) {
       public StorageObject call() {
         return storageRpc.copy(source, sourceOptions, target, targetOptions);
       }
-    }, retryParams, EXCEPTION_HANDLER));
+    }, options().retryParams(), EXCEPTION_HANDLER));
   }
 
   @Override
@@ -295,7 +357,7 @@ public byte[] load(String bucket, String blob, BlobSourceOption... options) {
       public byte[] call() {
         return storageRpc.load(storageObject, optionsMap);
       }
-    }, retryParams, EXCEPTION_HANDLER);
+    }, options().retryParams(), EXCEPTION_HANDLER);
   }
 
   @Override
@@ -361,6 +423,8 @@ private  List> transformBatch
 
   private static class BlobReadChannelImpl implements BlobReadChannel {
 
+    private static final long serialVersionUID = 1612561791239832259L;
+
     private final StorageServiceOptions serviceOptions;
     private final Blob blob;
     private final Map requestOptions;
@@ -369,7 +433,6 @@ private static class BlobReadChannelImpl implements BlobReadChannel {
     private boolean endOfStream;
 
     private transient StorageRpc storageRpc;
-    private transient RetryParams retryParams;
     private transient StorageObject storageObject;
     private transient int bufferPos;
     private transient byte[] buffer;
@@ -400,7 +463,6 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
 
     private void initTransients() {
       storageRpc = serviceOptions.storageRpc();
-      retryParams = firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries());
       storageObject = blob.toPb();
     }
 
@@ -445,7 +507,7 @@ public int read(ByteBuffer byteBuffer) throws IOException {
           public byte[] call() {
             return storageRpc.read(storageObject, requestOptions, position, toRead);
           }
-        }, retryParams, EXCEPTION_HANDLER);
+        }, serviceOptions.retryParams(), EXCEPTION_HANDLER);
         if (toRead > buffer.length) {
           endOfStream = true;
         }
@@ -472,6 +534,7 @@ private static class BlobWriterChannelImpl implements BlobWriteChannel {
 
     private static final int CHUNK_SIZE = 256 * 1024;
     private static final int COMPACT_THRESHOLD = (int) Math.round(CHUNK_SIZE * 0.8);
+    private static final long serialVersionUID = -4067648781804698786L;
 
     private final StorageServiceOptions options;
     private final Blob blob;
@@ -482,7 +545,6 @@ private static class BlobWriterChannelImpl implements BlobWriteChannel {
     private boolean isOpen = true;
 
     private transient StorageRpc storageRpc;
-    private transient RetryParams retryParams;
     private transient StorageObject storageObject;
 
     public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
@@ -515,7 +577,7 @@ private void flush() {
           public void run() {
             storageRpc.write(uploadId, buffer, 0, storageObject, position, length, false);
           }
-        }));
+        }), options.retryParams(), EXCEPTION_HANDLER);
         position += length;
         limit -= length;
         byte[] temp = new byte[CHUNK_SIZE];
@@ -536,7 +598,6 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
 
     private void initTransients() {
       storageRpc = options.storageRpc();
-      retryParams = firstNonNull(options.retryParams(), RetryParams.noRetries());
       storageObject = blob.toPb();
     }
 
@@ -575,7 +636,7 @@ public void close() throws IOException {
           public void run() {
             storageRpc.write(uploadId, buffer, 0, storageObject, position, limit, true);
           }
-        }));
+        }), options.retryParams(), EXCEPTION_HANDLER);
         position += buffer.length;
         isOpen = false;
         buffer = null;
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 13b14a1fbf76..87cac636fa92 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -33,6 +33,7 @@ public class StorageServiceOptions extends ServiceOptions {
@@ -68,10 +69,15 @@ protected Set scopes() {
   }
 
   StorageRpc storageRpc() {
+    if (storageRpc != null) {
+      return storageRpc;
+    }
     if (serviceRpcFactory() != null) {
-      return serviceRpcFactory().create(this);
+      storageRpc = serviceRpcFactory().create(this);
+    } else {
+      storageRpc = ServiceRpcProvider.storage(this);
     }
-    return ServiceRpcProvider.storage(this);
+    return storageRpc;
   }
 
   public String pathDelimiter() {

From 3dffcef113fe93bcce80dda0fd57103b39c76801 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Fri, 15 May 2015 21:44:15 -0700
Subject: [PATCH 229/732] fix test and add some javadoc

---
 src/main/java/com/google/gcloud/storage/ListResult.java     | 6 ++++++
 .../java/com/google/gcloud/storage/SerializationTest.java   | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/google/gcloud/storage/ListResult.java b/src/main/java/com/google/gcloud/storage/ListResult.java
index 55406f90cf0c..b6e36cd67954 100644
--- a/src/main/java/com/google/gcloud/storage/ListResult.java
+++ b/src/main/java/com/google/gcloud/storage/ListResult.java
@@ -41,10 +41,16 @@ public ListResult(NextPageFetcher pageFetcher, String cursor, Iterable res
     this.results = results;
   }
 
+  /**
+   * Returns the cursor for the nextPage or {@code null} if no more results.
+   */
   public String nextPageCursor() {
     return cursor;
   }
 
+  /**
+   * Returns the results of the nextPage or {@code null} if no more result.
+   */
   public ListResult nextPage() {
     if (cursor == null || pageFetcher == null) {
       return null;
diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/src/test/java/com/google/gcloud/storage/SerializationTest.java
index 365462d3f69a..83bc9e5b25d9 100644
--- a/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -51,7 +51,7 @@ public class SerializationTest {
       Collections.>emptyList(),
       Collections.>emptyList());
   private static final ListResult LIST_RESULT =
-      new ListResult<>("c", Collections.singletonList(Blob.of("b", "n")));
+      new ListResult<>(null, "c", Collections.singletonList(Blob.of("b", "n")));
   private static StorageService.BlobListOption BLOB_LIST_OPTIONS =
       StorageService.BlobListOption.maxResults(100);
   private static StorageService.BlobSourceOption BLOB_SOURCE_OPTIONS =

From 22b877b0dabb1ee80989e6d0477d7ed006793b5a Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Sun, 17 May 2015 18:59:52 -0700
Subject: [PATCH 230/732] extract reader/writer impl, increase their default
 buffer and minimize footprint when possible

---
 .../gcloud/examples/StorageExample.java       |   4 +-
 .../gcloud/storage/BlobReadChannelImpl.java   | 140 +++++++++++
 .../gcloud/storage/BlobWriterChannelImpl.java | 138 +++++++++++
 .../gcloud/storage/StorageServiceImpl.java    | 224 +-----------------
 4 files changed, 282 insertions(+), 224 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
 create mode 100644 src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index b6f54efa1d93..a4550f19d2de 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,6 +16,7 @@
 
 package com.google.gcloud.examples;
 
+import com.google.gcloud.RetryParams;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 import com.google.gcloud.storage.BatchRequest;
 import com.google.gcloud.storage.BatchResponse;
@@ -498,7 +499,8 @@ public static void main(String... args) throws Exception {
       printUsage();
       return;
     }
-    StorageServiceOptions.Builder optionsBuilder = StorageServiceOptions.builder();
+    StorageServiceOptions.Builder optionsBuilder =
+        StorageServiceOptions.builder().retryParams(RetryParams.getDefaultInstance());
     StorageAction action;
     if (args.length >= 2 && !ACTIONS.containsKey(args[0])) {
       optionsBuilder.projectId(args[0]);
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
new file mode 100644
index 000000000000..8ca8a01f2df3
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static com.google.gcloud.RetryHelper.runWithRetries;
+
+import com.google.api.services.storage.model.StorageObject;
+import com.google.gcloud.spi.StorageRpc;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+/**
+ * Default implementation for BlobReadChannel.
+ */
+class BlobReadChannelImpl implements BlobReadChannel {
+
+  private static final int MIN_BUFFER_SIZE = 2 * 1024 * 1024;
+  private static final long serialVersionUID = 4821762590742862669L;
+
+  private final StorageServiceOptions serviceOptions;
+  private final Blob blob;
+  private final Map requestOptions;
+  private int position;
+  private boolean isOpen;
+  private boolean endOfStream;
+
+  private transient StorageRpc storageRpc;
+  private transient StorageObject storageObject;
+  private transient int bufferPos;
+  private transient byte[] buffer;
+
+  BlobReadChannelImpl(StorageServiceOptions serviceOptions, Blob blob,
+      Map requestOptions) {
+    this.serviceOptions = serviceOptions;
+    this.blob = blob;
+    this.requestOptions = requestOptions;
+    isOpen = true;
+    initTransients();
+  }
+
+  private void writeObject(ObjectOutputStream out) throws IOException {
+    if (buffer != null) {
+      position += bufferPos;
+      buffer = null;
+      bufferPos = 0;
+      endOfStream = false;
+    }
+    out.defaultWriteObject();
+  }
+
+  private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+    in.defaultReadObject();
+    initTransients();
+  }
+
+  private void initTransients() {
+    storageRpc = serviceOptions.storageRpc();
+    storageObject = blob.toPb();
+  }
+
+  @Override
+  public boolean isOpen() {
+    return isOpen;
+  }
+
+  @Override
+  public void close() {
+    if (isOpen) {
+      buffer = null;
+      isOpen = false;
+    }
+  }
+
+  private void validateOpen() throws IOException {
+    if (!isOpen) {
+      throw new IOException("stream is closed");
+    }
+  }
+
+  @Override
+  public void seek(int position) throws IOException {
+    validateOpen();
+    this.position = position;
+    buffer = null;
+    bufferPos = 0;
+    endOfStream = false;
+  }
+
+  @Override
+  public int read(ByteBuffer byteBuffer) throws IOException {
+    validateOpen();
+    if (buffer == null) {
+      if (endOfStream) {
+        return -1;
+      }
+      final int toRead = Math.max(byteBuffer.remaining(), MIN_BUFFER_SIZE);
+      buffer = runWithRetries(new Callable() {
+        @Override
+        public byte[] call() {
+          return storageRpc.read(storageObject, requestOptions, position, toRead);
+        }
+      }, serviceOptions.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
+      if (toRead > buffer.length) {
+        endOfStream = true;
+        if (buffer.length == 0) {
+          buffer = null;
+          return -1;
+        }
+      }
+    }
+    int toWrite = Math.min(buffer.length - bufferPos, byteBuffer.remaining());
+    byteBuffer.put(buffer, bufferPos, toWrite);
+    bufferPos += toWrite;
+    if (bufferPos >= buffer.length) {
+      position += buffer.length;
+      buffer = null;
+      bufferPos = 0;
+    }
+    return toWrite;
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
new file mode 100644
index 000000000000..b7736346bba0
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+import static com.google.gcloud.RetryHelper.runWithRetries;
+import static java.util.concurrent.Executors.callable;
+
+import com.google.api.services.storage.model.StorageObject;
+import com.google.gcloud.spi.StorageRpc;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * Default implementation for BlobWriteChannel.
+ */
+class BlobWriterChannelImpl implements BlobWriteChannel {
+
+  private static final long serialVersionUID = 8675286882724938737L;
+  private static final int CHUNK_SIZE = 256 * 1024;
+  private static final int MIN_BUFFER_SIZE = 8 * CHUNK_SIZE;
+
+  private final StorageServiceOptions options;
+  private final Blob blob;
+  private final String uploadId;
+  private int position;
+  private byte[] buffer = new byte[0];
+  private int limit;
+  private boolean isOpen = true;
+
+  private transient StorageRpc storageRpc;
+  private transient StorageObject storageObject;
+
+  public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
+      Map optionsMap) {
+    this.options = options;
+    this.blob = blob;
+    initTransients();
+    uploadId = options.storageRpc().open(storageObject, optionsMap);
+  }
+
+  private void writeObject(ObjectOutputStream out) throws IOException {
+    if (isOpen) {
+      flush(true);
+    }
+    out.defaultWriteObject();
+  }
+
+  private void flush(boolean compact) {
+    if (limit >= MIN_BUFFER_SIZE || compact && limit >= CHUNK_SIZE) {
+      final int length = limit - limit % CHUNK_SIZE;
+      runWithRetries(callable(new Runnable() {
+        @Override
+        public void run() {
+          storageRpc.write(uploadId, buffer, 0, storageObject, position, length, false);
+        }
+      }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
+      position += length;
+      limit -= length;
+      byte[] temp = new byte[compact ? limit : MIN_BUFFER_SIZE];
+      System.arraycopy(buffer, length, temp, 0, limit);
+      buffer = temp;
+    }
+  }
+
+  private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+    in.defaultReadObject();
+    if (isOpen) {
+      initTransients();
+    }
+  }
+
+  private void initTransients() {
+    storageRpc = options.storageRpc();
+    storageObject = blob.toPb();
+  }
+
+  private void validateOpen() throws IOException {
+    if (!isOpen) {
+      throw new IOException("stream is closed");
+    }
+  }
+
+  @Override
+  public int write(ByteBuffer byteBuffer) throws IOException {
+    validateOpen();
+    int toWrite = byteBuffer.remaining();
+    int spaceInBuffer = buffer.length - limit;
+    if (spaceInBuffer >= toWrite) {
+      byteBuffer.get(buffer, limit, toWrite);
+    } else {
+      buffer = Arrays.copyOf(buffer,
+          Math.max(MIN_BUFFER_SIZE, buffer.length + toWrite - spaceInBuffer));
+      byteBuffer.get(buffer, limit, toWrite);
+    }
+    limit += toWrite;
+    flush(false);
+    return toWrite;
+  }
+
+  @Override
+  public boolean isOpen() {
+    return isOpen;
+  }
+
+  @Override
+  public void close() throws IOException {
+    if (isOpen) {
+      runWithRetries(callable(new Runnable() {
+        @Override
+        public void run() {
+          storageRpc.write(uploadId, buffer, 0, storageObject, position, limit, true);
+        }
+      }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
+      position += buffer.length;
+      isOpen = false;
+      buffer = null;
+    }
+  }
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 1f66401105d0..706cba123903 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -47,11 +47,7 @@
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -78,7 +74,7 @@ public RetryResult beforeEval(Exception exception) {
       return null;
     }
   };
-  private static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
+  static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder()
       .abortOn(RuntimeException.class).interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build();
   private static final byte[] EMPTY_BYTE_ARRAY = {};
 
@@ -359,230 +355,12 @@ private  List> transformBatch
     return response;
   }
 
-  private static class BlobReadChannelImpl implements BlobReadChannel {
-
-    private final StorageServiceOptions serviceOptions;
-    private final Blob blob;
-    private final Map requestOptions;
-    private int position;
-    private boolean isOpen;
-    private boolean endOfStream;
-
-    private transient StorageRpc storageRpc;
-    private transient RetryParams retryParams;
-    private transient StorageObject storageObject;
-    private transient int bufferPos;
-    private transient byte[] buffer;
-
-    BlobReadChannelImpl(StorageServiceOptions serviceOptions, Blob blob,
-        Map requestOptions) {
-      this.serviceOptions = serviceOptions;
-      this.blob = blob;
-      this.requestOptions = requestOptions;
-      isOpen = true;
-      initTransients();
-    }
-
-    private void writeObject(ObjectOutputStream out) throws IOException {
-      if (buffer != null) {
-        position += bufferPos;
-        buffer = null;
-        bufferPos = 0;
-        endOfStream = false;
-      }
-      out.defaultWriteObject();
-    }
-
-    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
-      in.defaultReadObject();
-      initTransients();
-    }
-
-    private void initTransients() {
-      storageRpc = serviceOptions.storageRpc();
-      retryParams = firstNonNull(serviceOptions.retryParams(), RetryParams.noRetries());
-      storageObject = blob.toPb();
-    }
-
-    @Override
-    public boolean isOpen() {
-      return isOpen;
-    }
-
-    @Override
-    public void close() {
-      if (isOpen) {
-        buffer = null;
-        isOpen = false;
-      }
-    }
-
-    private void validateOpen() throws IOException {
-      if (!isOpen) {
-        throw new IOException("stream is closed");
-      }
-    }
-
-    @Override
-    public void seek(int position) throws IOException {
-      validateOpen();
-      this.position = position;
-      buffer = null;
-      bufferPos = 0;
-      endOfStream = false;
-    }
-
-    @Override
-    public int read(ByteBuffer byteBuffer) throws IOException {
-      validateOpen();
-      if (buffer == null) {
-        if (endOfStream) {
-          return -1;
-        }
-        final int toRead = Math.max(byteBuffer.remaining(), 256 * 1024);
-        buffer = runWithRetries(new Callable() {
-          @Override
-          public byte[] call() {
-            return storageRpc.read(storageObject, requestOptions, position, toRead);
-          }
-        }, retryParams, EXCEPTION_HANDLER);
-        if (toRead > buffer.length) {
-          endOfStream = true;
-        }
-      }
-      int toWrite = Math.min(buffer.length - bufferPos, byteBuffer.remaining());
-      byteBuffer.put(buffer, bufferPos, toWrite);
-      bufferPos += toWrite;
-      if (bufferPos >= buffer.length) {
-        position += buffer.length;
-        buffer = null;
-        bufferPos = 0;
-      }
-      return toWrite;
-    }
-  }
-
   @Override
   public BlobReadChannel reader(String bucket, String blob, BlobSourceOption... options) {
     Map optionsMap = optionMap(options);
     return new BlobReadChannelImpl(options(), Blob.of(bucket, blob), optionsMap);
   }
 
-  private static class BlobWriterChannelImpl implements BlobWriteChannel {
-
-    private static final int CHUNK_SIZE = 256 * 1024;
-    private static final int COMPACT_THRESHOLD = (int) Math.round(CHUNK_SIZE * 0.8);
-
-    private final StorageServiceOptions options;
-    private final Blob blob;
-    private final String uploadId;
-    private int position;
-    private byte[] buffer = new byte[CHUNK_SIZE];
-    private int limit;
-    private boolean isOpen = true;
-
-    private transient StorageRpc storageRpc;
-    private transient RetryParams retryParams;
-    private transient StorageObject storageObject;
-
-    public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
-        Map optionsMap) {
-      this.options = options;
-      this.blob = blob;
-      initTransients();
-      uploadId = options.storageRpc().open(storageObject, optionsMap);
-    }
-
-    private void writeObject(ObjectOutputStream out) throws IOException {
-      if (!isOpen) {
-        out.defaultWriteObject();
-        return;
-      }
-      flush();
-      byte[] temp = buffer;
-      if (limit < COMPACT_THRESHOLD) {
-        buffer = Arrays.copyOf(buffer, limit);
-      }
-      out.defaultWriteObject();
-      buffer = temp;
-    }
-
-    private void flush() {
-      if (limit >= CHUNK_SIZE) {
-        final int length = limit - limit % CHUNK_SIZE;
-        runWithRetries(callable(new Runnable() {
-          @Override
-          public void run() {
-            storageRpc.write(uploadId, buffer, 0, storageObject, position, length, false);
-          }
-        }));
-        position += length;
-        limit -= length;
-        byte[] temp = new byte[CHUNK_SIZE];
-        System.arraycopy(buffer, length, temp, 0, limit);
-        buffer = temp;
-      }
-    }
-
-    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
-      in.defaultReadObject();
-      if (isOpen) {
-        if (buffer.length < CHUNK_SIZE) {
-          buffer = Arrays.copyOf(buffer, CHUNK_SIZE);
-        }
-        initTransients();
-      }
-    }
-
-    private void initTransients() {
-      storageRpc = options.storageRpc();
-      retryParams = firstNonNull(options.retryParams(), RetryParams.noRetries());
-      storageObject = blob.toPb();
-    }
-
-    private void validateOpen() throws IOException {
-      if (!isOpen) {
-        throw new IOException("stream is closed");
-      }
-    }
-
-    @Override
-    public int write(ByteBuffer byteBuffer) throws IOException {
-      validateOpen();
-      int toWrite = byteBuffer.remaining();
-      int spaceInBuffer = buffer.length - limit;
-      if (spaceInBuffer >= toWrite) {
-        byteBuffer.get(buffer, limit, toWrite);
-      } else {
-        buffer = Arrays.copyOf(buffer, buffer.length + toWrite - spaceInBuffer);
-        byteBuffer.get(buffer, limit, toWrite);
-      }
-      limit += toWrite;
-      flush();
-      return toWrite;
-    }
-
-    @Override
-    public boolean isOpen() {
-      return isOpen;
-    }
-
-    @Override
-    public void close() throws IOException {
-      if (isOpen) {
-        runWithRetries(callable(new Runnable() {
-          @Override
-          public void run() {
-            storageRpc.write(uploadId, buffer, 0, storageObject, position, limit, true);
-          }
-        }));
-        position += buffer.length;
-        isOpen = false;
-        buffer = null;
-      }
-    }
-  }
-
   @Override
   public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
     final Map optionsMap = optionMap(blob, options);

From fbaf57133e83caaf5e4be2866c8935ffc808390a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Mon, 18 May 2015 10:56:25 -0700
Subject: [PATCH 231/732] fix merge conflict

---
 .../com/google/gcloud/storage/ListResultTest.java | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/test/java/com/google/gcloud/storage/ListResultTest.java b/src/test/java/com/google/gcloud/storage/ListResultTest.java
index 1345b0ced240..8a2e69d0c084 100644
--- a/src/test/java/com/google/gcloud/storage/ListResultTest.java
+++ b/src/test/java/com/google/gcloud/storage/ListResultTest.java
@@ -22,13 +22,26 @@
 
 import org.junit.Test;
 
+import java.util.Collections;
+
 public class ListResultTest {
 
   @Test
   public void testListResult() throws Exception {
     ImmutableList values = ImmutableList.of("1", "2");
-    ListResult result = new ListResult("c", values);
+    final ListResult nextResult =
+        new ListResult<>(null, "c", Collections.emptyList());
+    ListResult.NextPageFetcher fetcher = new ListResult.NextPageFetcher() {
+
+      @Override
+      public ListResult nextPage() {
+        return nextResult;
+      }
+    };
+    ListResult result = new ListResult(fetcher, "c", values);
+    assertEquals(nextResult, result.nextPage());
     assertEquals("c", result.nextPageCursor());
     assertEquals(values, ImmutableList.copyOf(result.iterator()));
+
   }
 }

From dd33dd7f02a071d3860b65b292ea5d946fb61d54 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 20 May 2015 14:49:34 -0700
Subject: [PATCH 232/732] make chunk size configurable

---
 .../gcloud/examples/StorageExample.java       |  2 +-
 .../gcloud/storage/BlobReadChannel.java       |  7 +++++++
 .../gcloud/storage/BlobReadChannelImpl.java   | 10 ++++++++--
 .../gcloud/storage/BlobWriteChannel.java      |  5 +++++
 .../gcloud/storage/BlobWriterChannelImpl.java | 20 ++++++++++++-------
 .../google/gcloud/storage/StorageService.java |  4 ++--
 6 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index a4550f19d2de..b0d44c292d2c 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -479,7 +479,7 @@ public String params() {
   }
 
   public static void printUsage() {
-    StringBuilder actionAndParams = new StringBuilder("");
+    StringBuilder actionAndParams = new StringBuilder();
     for (Map.Entry entry : ACTIONS.entrySet()) {
       actionAndParams.append("\n\t").append(entry.getKey());
 
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
index 89f3420a2a28..ad1a385d9a83 100644
--- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
@@ -39,4 +39,11 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos
   void close();
 
   void seek(int position) throws IOException;
+
+  /**
+   * Sets the minimum size that will be read by a single RPC.
+   * Read data will be locally buffered until consumed.
+   */
+  void chunkSize(int chunkSize);
+
 }
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
index 8ca8a01f2df3..27d37b127d55 100644
--- a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
@@ -33,7 +33,7 @@
  */
 class BlobReadChannelImpl implements BlobReadChannel {
 
-  private static final int MIN_BUFFER_SIZE = 2 * 1024 * 1024;
+  private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
   private static final long serialVersionUID = 4821762590742862669L;
 
   private final StorageServiceOptions serviceOptions;
@@ -42,6 +42,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private int position;
   private boolean isOpen;
   private boolean endOfStream;
+  private int chunkSize = DEFAULT_CHUNK_SIZE;
 
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
@@ -105,6 +106,11 @@ public void seek(int position) throws IOException {
     endOfStream = false;
   }
 
+  @Override
+  public void chunkSize(int chunkSize) {
+    this.chunkSize = chunkSize <= 0 ? DEFAULT_CHUNK_SIZE : chunkSize;
+  }
+
   @Override
   public int read(ByteBuffer byteBuffer) throws IOException {
     validateOpen();
@@ -112,7 +118,7 @@ public int read(ByteBuffer byteBuffer) throws IOException {
       if (endOfStream) {
         return -1;
       }
-      final int toRead = Math.max(byteBuffer.remaining(), MIN_BUFFER_SIZE);
+      final int toRead = Math.max(byteBuffer.remaining(), chunkSize);
       buffer = runWithRetries(new Callable() {
         @Override
         public byte[] call() {
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
index 77ce84a8ea7a..20b2ce087632 100644
--- a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
@@ -29,4 +29,9 @@
  */
 public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable {
 
+  /**
+   * Sets the minimum size that will be written by a single RPC.
+   * Written data will be buffered and only flushed upon reaching this size or closing the channel.
+   */
+  void chunkSize(int chunkSize);
 }
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
index b7736346bba0..2b8e66cc33ce 100644
--- a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
+++ b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
@@ -35,8 +35,8 @@
 class BlobWriterChannelImpl implements BlobWriteChannel {
 
   private static final long serialVersionUID = 8675286882724938737L;
-  private static final int CHUNK_SIZE = 256 * 1024;
-  private static final int MIN_BUFFER_SIZE = 8 * CHUNK_SIZE;
+  private static final int MIN_CHUNK_SIZE = 256 * 1024;
+  private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE;
 
   private final StorageServiceOptions options;
   private final Blob blob;
@@ -45,6 +45,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private byte[] buffer = new byte[0];
   private int limit;
   private boolean isOpen = true;
+  private int chunkSize = DEFAULT_CHUNK_SIZE;
 
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
@@ -65,8 +66,8 @@ private void writeObject(ObjectOutputStream out) throws IOException {
   }
 
   private void flush(boolean compact) {
-    if (limit >= MIN_BUFFER_SIZE || compact && limit >= CHUNK_SIZE) {
-      final int length = limit - limit % CHUNK_SIZE;
+    if (limit >= chunkSize || compact && limit >= MIN_CHUNK_SIZE) {
+      final int length = limit - limit % MIN_CHUNK_SIZE;
       runWithRetries(callable(new Runnable() {
         @Override
         public void run() {
@@ -75,7 +76,7 @@ public void run() {
       }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
       position += length;
       limit -= length;
-      byte[] temp = new byte[compact ? limit : MIN_BUFFER_SIZE];
+      byte[] temp = new byte[compact ? limit : chunkSize];
       System.arraycopy(buffer, length, temp, 0, limit);
       buffer = temp;
     }
@@ -107,8 +108,7 @@ public int write(ByteBuffer byteBuffer) throws IOException {
     if (spaceInBuffer >= toWrite) {
       byteBuffer.get(buffer, limit, toWrite);
     } else {
-      buffer = Arrays.copyOf(buffer,
-          Math.max(MIN_BUFFER_SIZE, buffer.length + toWrite - spaceInBuffer));
+      buffer = Arrays.copyOf(buffer, Math.max(chunkSize, buffer.length + toWrite - spaceInBuffer));
       byteBuffer.get(buffer, limit, toWrite);
     }
     limit += toWrite;
@@ -135,4 +135,10 @@ public void run() {
       buffer = null;
     }
   }
+
+  @Override
+  public void chunkSize(int chunkSize) {
+    chunkSize = (chunkSize / MIN_CHUNK_SIZE) * MIN_CHUNK_SIZE;
+    this.chunkSize = Math.max(MIN_CHUNK_SIZE, chunkSize);
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 9e95b0f1281d..6fefb3af3b16 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -121,8 +121,8 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
       return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
 
-    public static BlobTargetOption doesNotExists() {
-      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0);
+    public static BlobTargetOption doesNotExist() {
+      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L);
     }
 
     public static BlobTargetOption generationMatch() {

From fc88158961e9d3e38bd7011140314470319afe03 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 20 May 2015 22:50:24 -0700
Subject: [PATCH 233/732] work on signURL

---
 .../com/google/gcloud/AuthCredentials.java    |  12 +-
 .../java/com/google/gcloud/storage/Cors.java  |  18 +--
 .../com/google/gcloud/storage/HttpMethod.java |  24 +++
 .../google/gcloud/storage/StorageService.java | 148 ++++++++++++++++++
 .../gcloud/storage/StorageServiceImpl.java    |   8 +-
 .../com/google/gcloud/storage/CorsTest.java   |   3 +-
 6 files changed, 197 insertions(+), 16 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/HttpMethod.java

diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java
index 839da54e62cf..6cdb737ddd91 100644
--- a/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -62,7 +62,7 @@ private Object readResolve() throws ObjectStreamException {
     }
   }
 
-  private static class ServiceAccountAuthCredentials extends AuthCredentials {
+  public static class ServiceAccountAuthCredentials extends AuthCredentials {
 
     private static final long serialVersionUID = 8007708734318445901L;
     private final String account;
@@ -94,6 +94,14 @@ protected HttpRequestInitializer httpRequestInitializer(
       return builder.build();
     }
 
+    public String account() {
+      return account;
+    }
+
+    public PrivateKey privateKey() {
+      return privateKey;
+    }
+
     @Override
     public int hashCode() {
       return Objects.hash(account, privateKey);
@@ -187,7 +195,7 @@ public static AuthCredentials createApplicationDefaults() throws IOException {
     return new ApplicationDefaultAuthCredentials();
   }
 
-  public static AuthCredentials createFor(String account, PrivateKey privateKey) {
+  public static ServiceAccountAuthCredentials createFor(String account, PrivateKey privateKey) {
     return new ServiceAccountAuthCredentials(account, privateKey);
   }
 
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index b1953aa5e0e4..ce8cfb95b6e9 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -53,14 +53,10 @@ public Bucket.Cors apply(Cors cors) {
   };
 
   private final Integer maxAgeSeconds;
-  private final ImmutableList methods;
+  private final ImmutableList methods;
   private final ImmutableList origins;
   private final ImmutableList responseHeaders;
 
-  public enum Method {
-    ANY, GET, HEAD, PUT, POST, DELETE
-  }
-
   public static final class Origin implements Serializable {
 
     private static final long serialVersionUID = -4447958124895577993L;
@@ -118,7 +114,7 @@ public String value() {
   public static final class Builder {
 
     private Integer maxAgeSeconds;
-    private ImmutableList methods;
+    private ImmutableList methods;
     private ImmutableList origins;
     private ImmutableList responseHeaders;
 
@@ -129,7 +125,7 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) {
       return this;
     }
 
-    public Builder methods(Iterable methods) {
+    public Builder methods(Iterable methods) {
       this.methods = methods != null ? ImmutableList.copyOf(methods) : null;
       return this;
     }
@@ -160,7 +156,7 @@ public Integer maxAgeSeconds() {
     return maxAgeSeconds;
   }
 
-  public List methods() {
+  public List methods() {
     return methods;
   }
 
@@ -217,10 +213,10 @@ Bucket.Cors toPb() {
   static Cors fromPb(Bucket.Cors cors) {
     Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds());
     if (cors.getMethod() != null) {
-      builder.methods(transform(cors.getMethod(), new Function() {
+      builder.methods(transform(cors.getMethod(), new Function() {
         @Override
-        public Method apply(String name) {
-          return Method.valueOf(name.toUpperCase());
+        public HttpMethod apply(String name) {
+          return HttpMethod.valueOf(name.toUpperCase());
         }
       }));
     }
diff --git a/src/main/java/com/google/gcloud/storage/HttpMethod.java b/src/main/java/com/google/gcloud/storage/HttpMethod.java
new file mode 100644
index 000000000000..f5889aedae90
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/HttpMethod.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+/**
+ *
+ */
+public enum HttpMethod {
+  GET, HEAD, PUT, POST, DELETE
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 6fefb3af3b16..2e500b001a93 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -16,14 +16,19 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.collect.ImmutableList;
+import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.Service;
 import com.google.gcloud.spi.StorageRpc;
 
+import org.joda.time.DateTime;
+
 import java.io.Serializable;
+import java.net.URL;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashSet;
@@ -407,6 +412,139 @@ public static Builder builder() {
     }
   }
 
+  /**
+   * A request for signing a URL.
+   */
+  class SignUrlRequest implements Serializable {
+
+    private final Blob blob;
+    private final HttpMethod httpMethod;
+    private final Long expiration;
+    private final boolean includeContentType;
+    private final boolean includeMd5;
+    private final String headers;
+    private final ServiceAccountAuthCredentials authCredentials;
+
+    public static class Builder {
+
+      private Blob blob;
+      private HttpMethod httpMethod;
+      private long expiration;
+      private boolean includeContentType;
+      private boolean includeMd5;
+      private String headers;
+      private ServiceAccountAuthCredentials authCredentials;
+
+      private Builder() {}
+
+      public Builder blob(Blob blob) {
+        this.blob = blob;
+        return this;
+      }
+
+      /**
+       * The HTTP method to be used with the signed URL.
+       */
+      public Builder httpMethod(HttpMethod httpMethod) {
+        this.httpMethod = httpMethod;
+        return this;
+      }
+
+      /**
+       * Sets expiration time for the URL.
+       * Defaults to one day.
+       */
+      public Builder expiration(long expiration) {
+        this.expiration = expiration;
+        return this;
+      }
+
+      /**
+       * Indicate if signature should include the blob's content-type.
+       * If {@code true} users of the signed URL should include
+       * the same content-type with their request.
+       */
+      public Builder includeContentType(boolean includeContentType) {
+        this.includeContentType = includeContentType;
+        return this;
+      }
+
+      /**
+       * Indicate if signature should include the blob's md5.
+       * If {@code true} users of the signed URL should include it with their requests.
+       */
+      public Builder includeMd5(boolean includeMd5) {
+        this.includeMd5 = includeMd5;
+        return this;
+      }
+
+      /**
+       *  If headers are provided, the server will check to make sure that the client
+       *  provides matching values.
+       *  For information about how to create canonical headers for signing,
+       *  see About Canonical Extension Headers.
+       */
+      public Builder canonicalizedExtensionHeaders(String headers) {
+        this.headers = headers;
+        return this;
+      }
+
+      /**
+       * The service account credentials for signing the URL.
+       */
+      public Builder serviceAccountAuthCredentials(ServiceAccountAuthCredentials authCredentials) {
+        this.authCredentials = authCredentials;
+        return this;
+      }
+
+      public SignUrlRequest build() {
+        return new SignUrlRequest(this);
+      }
+    }
+
+    private SignUrlRequest(Builder builder) {
+      blob = checkNotNull(builder.blob);
+      httpMethod = builder.httpMethod;
+      expiration = firstNonNull(builder.expiration, new DateTime().plusDays(1).getMillis());
+      includeContentType = builder.includeContentType; // verify blob has content-type
+      includeMd5 = builder.includeMd5; // verify blob  has md5
+      authCredentials = builder.authCredentials; // default if null
+      headers = builder.headers;
+    }
+
+    public Blob blob() {
+      return blob;
+    }
+
+    public HttpMethod httpMethod() {
+      return httpMethod;
+    }
+
+    public String canonicalizedExtensionHeaders() {
+      return headers;
+    }
+
+    public long expiration() {
+      return expiration;
+    }
+
+    public ServiceAccountAuthCredentials authCredentials() {
+      return authCredentials;
+    }
+
+    public boolean includeContentType() {
+      return includeContentType;
+    }
+
+    public boolean includeMd5() {
+      return includeMd5;
+    }
+
+    public static Builder builder() {
+      return new Builder();
+    }
+  }
+
   /**
    * Create a new bucket.
    *
@@ -528,4 +666,14 @@ public static Builder builder() {
    * @throws StorageServiceException upon failure
    */
   BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
+
+  /**
+   * Generates a signed URL for a blob.
+   * If you have a blob that you want to allow access to for a set
+   * amount of time, you can use this method to generate a URL that
+   * is only valid within a certain time period.
+   * This is particularly useful if you don't want publicly
+   * accessible blobs, but don't want to require users to explicitly log in.
+   */
+  URL signUrl(SignUrlRequest request);
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 95b95141be14..8dda8004c016 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -29,7 +29,6 @@
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-import static java.util.concurrent.Executors.callable;
 
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Function;
@@ -47,6 +46,7 @@
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
 import java.io.Serializable;
+import java.net.URL;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -429,6 +429,12 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
     return new BlobWriterChannelImpl(options(), blob, optionsMap);
   }
 
+  @Override
+  public URL signUrl(SignUrlRequest request) {
+    // todo: implement and add test
+    return null;
+  }
+
   private Map optionMap(Long generation, Long metaGeneration,
       Iterable options) {
     return optionMap(generation, metaGeneration, options, false);
diff --git a/src/test/java/com/google/gcloud/storage/CorsTest.java b/src/test/java/com/google/gcloud/storage/CorsTest.java
index 8b0379f03583..f29020c6380f 100644
--- a/src/test/java/com/google/gcloud/storage/CorsTest.java
+++ b/src/test/java/com/google/gcloud/storage/CorsTest.java
@@ -19,7 +19,6 @@
 import static org.junit.Assert.assertEquals;
 
 import com.google.common.collect.ImmutableList;
-import com.google.gcloud.storage.Cors.Method;
 import com.google.gcloud.storage.Cors.Origin;
 
 import org.junit.Test;
@@ -39,7 +38,7 @@ public void testOrigin() {
   public void corsTest() {
     List origins = ImmutableList.of(Origin.any(), Origin.of("o"));
     List headers = ImmutableList.of("h1", "h2");
-    List methods = ImmutableList.of(Method.ANY);
+    List methods = ImmutableList.of(HttpMethod.ANY);
     Cors cors = Cors.builder()
         .maxAgeSeconds(100)
         .origins(origins)

From 243e0804a7e17edf20b57007c54b7423784c7ab7 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 22 May 2015 16:01:32 -0700
Subject: [PATCH 234/732] initial work on modules

---
 RobustaSettings.xml                           |   2 -
 gcloud-java-core/README.md                    |  55 ++
 gcloud-java-core/pom.xml                      |  94 ++++
 .../com/google/gcloud/AuthCredentials.java    |   0
 .../java/com/google/gcloud/BaseService.java   |   0
 .../com/google/gcloud/ExceptionHandler.java   |   0
 .../java/com/google/gcloud/RetryHelper.java   |   0
 .../java/com/google/gcloud/RetryParams.java   |   0
 .../main/java/com/google/gcloud/Service.java  |   0
 .../com/google/gcloud/ServiceOptions.java     |   0
 .../google/gcloud/spi/ServiceRpcFactory.java  |   0
 .../google/gcloud/spi/ServiceRpcProvider.java |  32 ++
 .../google/gcloud/ExceptionHandlerTest.java   |   0
 .../com/google/gcloud/RetryHelperTest.java    |   0
 .../com/google/gcloud/RetryParamsTest.java    |   0
 gcloud-java-datastore/README.md               | 106 ++++
 gcloud-java-datastore/pom.xml                 |  44 ++
 .../datastore/BaseDatastoreBatchWriter.java   |   0
 .../google/gcloud/datastore/BaseEntity.java   |   0
 .../com/google/gcloud/datastore/BaseKey.java  |   0
 .../com/google/gcloud/datastore/Batch.java    |   0
 .../google/gcloud/datastore/BatchImpl.java    |   0
 .../google/gcloud/datastore/BatchOption.java  |   0
 .../com/google/gcloud/datastore/Blob.java     |   0
 .../google/gcloud/datastore/BlobValue.java    |   0
 .../google/gcloud/datastore/BooleanValue.java |   0
 .../com/google/gcloud/datastore/Cursor.java   |   0
 .../datastore/DatastoreBatchWriter.java       |   0
 .../gcloud/datastore/DatastoreHelper.java     |   0
 .../gcloud/datastore/DatastoreReader.java     |   0
 .../datastore/DatastoreReaderWriter.java      |   0
 .../gcloud/datastore/DatastoreService.java    |   0
 .../datastore/DatastoreServiceException.java  |   0
 .../datastore/DatastoreServiceFactory.java    |   0
 .../datastore/DatastoreServiceImpl.java       |   0
 .../datastore/DatastoreServiceOptions.java    |   7 +-
 .../gcloud/datastore/DatastoreWriter.java     |   0
 .../com/google/gcloud/datastore/DateTime.java |   0
 .../gcloud/datastore/DateTimeValue.java       |   0
 .../google/gcloud/datastore/DoubleValue.java  |   0
 .../com/google/gcloud/datastore/Entity.java   |   0
 .../google/gcloud/datastore/EntityValue.java  |   0
 .../google/gcloud/datastore/FullEntity.java   |   0
 .../com/google/gcloud/datastore/GqlQuery.java |   0
 .../gcloud/datastore/IncompleteKey.java       |   0
 .../java/com/google/gcloud/datastore/Key.java |   0
 .../google/gcloud/datastore/KeyFactory.java   |   0
 .../com/google/gcloud/datastore/KeyValue.java |   0
 .../google/gcloud/datastore/ListValue.java    |   0
 .../google/gcloud/datastore/LongValue.java    |   0
 .../google/gcloud/datastore/NullValue.java    |   0
 .../google/gcloud/datastore/PathElement.java  |   0
 .../gcloud/datastore/ProjectionEntity.java    |   0
 .../com/google/gcloud/datastore/Query.java    |   0
 .../google/gcloud/datastore/QueryResults.java |   0
 .../gcloud/datastore/QueryResultsImpl.java    |   0
 .../com/google/gcloud/datastore/RawValue.java |   0
 .../google/gcloud/datastore/Serializable.java |   0
 .../google/gcloud/datastore/StringValue.java  |   0
 .../gcloud/datastore/StructuredQuery.java     |   0
 .../google/gcloud/datastore/Transaction.java  |   0
 .../gcloud/datastore/TransactionImpl.java     |   0
 .../gcloud/datastore/TransactionOption.java   |   0
 .../google/gcloud/datastore/Validator.java    |   0
 .../com/google/gcloud/datastore/Value.java    |   0
 .../google/gcloud/datastore/ValueBuilder.java |   0
 .../gcloud/datastore/ValueMarshaller.java     |   0
 .../google/gcloud/datastore/ValueType.java    |   0
 .../google/gcloud/datastore/package-info.java |   0
 .../com/google/gcloud/spi/DatastoreRpc.java   |   0
 .../gcloud/spi/DatastoreRpcFactory.java       |  24 +
 .../gcloud/spi/DefaultDatastoreRpc.java       | 151 +++++
 .../BaseDatastoreBatchWriterTest.java         |   0
 .../gcloud/datastore/BaseEntityTest.java      |   0
 .../google/gcloud/datastore/BaseKeyTest.java  |   0
 .../com/google/gcloud/datastore/BlobTest.java |   0
 .../gcloud/datastore/BlobValueTest.java       |   0
 .../gcloud/datastore/BooleanValueTest.java    |   0
 .../google/gcloud/datastore/CursorTest.java   |   0
 .../gcloud/datastore/DatastoreHelperTest.java |   0
 .../DatastoreServiceExceptionTest.java        |   0
 .../DatastoreServiceOptionsTest.java          |   0
 .../datastore/DatastoreServiceTest.java       |   0
 .../google/gcloud/datastore/DateTimeTest.java |   0
 .../gcloud/datastore/DateTimeValueTest.java   |   0
 .../gcloud/datastore/DoubleValueTest.java     |   0
 .../google/gcloud/datastore/EntityTest.java   |   0
 .../gcloud/datastore/EntityValueTest.java     |   0
 .../gcloud/datastore/FullEntityTest.java      |   0
 .../gcloud/datastore/IncompleteKeyTest.java   |   0
 .../gcloud/datastore/KeyFactoryTest.java      |   0
 .../com/google/gcloud/datastore/KeyTest.java  |   0
 .../google/gcloud/datastore/KeyValueTest.java |   0
 .../gcloud/datastore/ListValueTest.java       |   0
 .../gcloud/datastore/LocalGcdHelper.java      |   0
 .../gcloud/datastore/LongValueTest.java       |   0
 .../gcloud/datastore/NullValueTest.java       |   0
 .../gcloud/datastore/PathElementTest.java     |   0
 .../datastore/ProjectionEntityTest.java       |   0
 .../google/gcloud/datastore/RawValueTest.java |   0
 .../gcloud/datastore/SerializationTest.java   |   0
 .../gcloud/datastore/StringValueTest.java     |   0
 .../google/gcloud/datastore/ValueTest.java    |   0
 gcloud-java-examples/README.md                |  55 ++
 gcloud-java-examples/pom.xml                  |  37 ++
 .../gcloud/examples/DatastoreExample.java     |   0
 .../gcloud/examples/StorageExample.java       |   0
 gcloud-java-storage/README.md                 |  63 +++
 gcloud-java-storage/pom.xml                   |  39 ++
 .../google/gcloud/spi/DefaultStorageRpc.java  | 525 ++++++++++++++++++
 .../com/google/gcloud/spi/StorageRpc.java     |   0
 .../google/gcloud/spi/StorageRpcFactory.java  |  23 +
 .../java/com/google/gcloud/storage/Acl.java   |   0
 .../google/gcloud/storage/BatchRequest.java   |   0
 .../google/gcloud/storage/BatchResponse.java  |   0
 .../java/com/google/gcloud/storage/Blob.java  |   0
 .../gcloud/storage/BlobReadChannel.java       |   0
 .../gcloud/storage/BlobReadChannelImpl.java   |   0
 .../gcloud/storage/BlobWriteChannel.java      |   0
 .../gcloud/storage/BlobWriterChannelImpl.java |   0
 .../com/google/gcloud/storage/Bucket.java     |   0
 .../java/com/google/gcloud/storage/Cors.java  |   0
 .../com/google/gcloud/storage/ListResult.java |   0
 .../com/google/gcloud/storage/Option.java     |   0
 .../google/gcloud/storage/StorageService.java |   0
 .../storage/StorageServiceException.java      |   0
 .../gcloud/storage/StorageServiceFactory.java |   0
 .../gcloud/storage/StorageServiceImpl.java    |   0
 .../gcloud/storage/StorageServiceOptions.java |   7 +-
 .../google/gcloud/storage/package-info.java   |   0
 .../com/google/gcloud/storage/AclTest.java    |   0
 .../gcloud/storage/BatchRequestTest.java      |   0
 .../gcloud/storage/BatchResponseTest.java     |   0
 .../com/google/gcloud/storage/BlobTest.java   |   0
 .../com/google/gcloud/storage/BucketTest.java |   0
 .../com/google/gcloud/storage/CorsTest.java   |   0
 .../google/gcloud/storage/ListResultTest.java |   0
 .../com/google/gcloud/storage/OptionTest.java |   0
 .../gcloud/storage/SerializationTest.java     |   0
 gcloud-java/README.md                         |  73 +++
 gcloud-java/pom.xml                           |  47 ++
 pom.xml                                       | 195 ++-----
 .../gcloud/spi/DatastoreRpcFactory.java       |   1 -
 .../gcloud/spi/DefaultDatastoreRpc.java       |   1 -
 .../google/gcloud/spi/DefaultStorageRpc.java  |   1 -
 .../google/gcloud/spi/ServiceRpcProvider.java |   1 -
 .../google/gcloud/spi/StorageRpcFactory.java  |   1 -
 src/site/apt/index.apt                        |   4 +-
 src/site/site.xml                             |   4 +-
 149 files changed, 1440 insertions(+), 152 deletions(-)
 delete mode 100644 RobustaSettings.xml
 create mode 100644 gcloud-java-core/README.md
 create mode 100644 gcloud-java-core/pom.xml
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/AuthCredentials.java (100%)
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/BaseService.java (100%)
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/ExceptionHandler.java (100%)
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/RetryHelper.java (100%)
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/RetryParams.java (100%)
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/Service.java (100%)
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/ServiceOptions.java (100%)
 rename {src => gcloud-java-core/src}/main/java/com/google/gcloud/spi/ServiceRpcFactory.java (100%)
 create mode 100644 gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
 rename {src => gcloud-java-core/src}/test/java/com/google/gcloud/ExceptionHandlerTest.java (100%)
 rename {src => gcloud-java-core/src}/test/java/com/google/gcloud/RetryHelperTest.java (100%)
 rename {src => gcloud-java-core/src}/test/java/com/google/gcloud/RetryParamsTest.java (100%)
 create mode 100644 gcloud-java-datastore/README.md
 create mode 100644 gcloud-java-datastore/pom.xml
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/BaseEntity.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/BaseKey.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Batch.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/BatchImpl.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/BatchOption.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Blob.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/BlobValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/BooleanValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Cursor.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreHelper.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreReader.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreService.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreServiceException.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java (95%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DatastoreWriter.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DateTime.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DateTimeValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/DoubleValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Entity.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/EntityValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/FullEntity.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/GqlQuery.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/IncompleteKey.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Key.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/KeyFactory.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/KeyValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/ListValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/LongValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/NullValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/PathElement.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/ProjectionEntity.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Query.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/QueryResults.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/QueryResultsImpl.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/RawValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Serializable.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/StringValue.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/StructuredQuery.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Transaction.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/TransactionImpl.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/TransactionOption.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Validator.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/Value.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/ValueBuilder.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/ValueMarshaller.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/ValueType.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/datastore/package-info.java (100%)
 rename {src => gcloud-java-datastore/src}/main/java/com/google/gcloud/spi/DatastoreRpc.java (100%)
 create mode 100644 gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
 create mode 100644 gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/BaseEntityTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/BaseKeyTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/BlobTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/BlobValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/BooleanValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/CursorTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/DateTimeTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/DateTimeValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/DoubleValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/EntityTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/EntityValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/FullEntityTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/KeyFactoryTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/KeyTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/KeyValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/ListValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/LocalGcdHelper.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/LongValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/NullValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/PathElementTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/RawValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/SerializationTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/StringValueTest.java (100%)
 rename {src => gcloud-java-datastore/src}/test/java/com/google/gcloud/datastore/ValueTest.java (100%)
 create mode 100644 gcloud-java-examples/README.md
 create mode 100644 gcloud-java-examples/pom.xml
 rename {src => gcloud-java-examples/src}/main/java/com/google/gcloud/examples/DatastoreExample.java (100%)
 rename {src => gcloud-java-examples/src}/main/java/com/google/gcloud/examples/StorageExample.java (100%)
 create mode 100644 gcloud-java-storage/README.md
 create mode 100644 gcloud-java-storage/pom.xml
 create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/spi/StorageRpc.java (100%)
 create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/Acl.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/BatchRequest.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/BatchResponse.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/Blob.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/BlobReadChannel.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/BlobWriteChannel.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/Bucket.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/Cors.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/ListResult.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/Option.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/StorageService.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/StorageServiceException.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/StorageServiceFactory.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/StorageServiceImpl.java (100%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/StorageServiceOptions.java (92%)
 rename {src => gcloud-java-storage/src}/main/java/com/google/gcloud/storage/package-info.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/AclTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/BatchRequestTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/BatchResponseTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/BlobTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/BucketTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/CorsTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/ListResultTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/OptionTest.java (100%)
 rename {src => gcloud-java-storage/src}/test/java/com/google/gcloud/storage/SerializationTest.java (100%)
 create mode 100644 gcloud-java/README.md
 create mode 100644 gcloud-java/pom.xml
 delete mode 100644 src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
 delete mode 100644 src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
 delete mode 100644 src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
 delete mode 100644 src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
 delete mode 100644 src/main/java/com/google/gcloud/spi/StorageRpcFactory.java

diff --git a/RobustaSettings.xml b/RobustaSettings.xml
deleted file mode 100644
index 8c664c57feb7..000000000000
--- a/RobustaSettings.xml
+++ /dev/null
@@ -1,2 +0,0 @@
-
-
diff --git a/gcloud-java-core/README.md b/gcloud-java-core/README.md
new file mode 100644
index 000000000000..e22b159d216f
--- /dev/null
+++ b/gcloud-java-core/README.md
@@ -0,0 +1,55 @@
+Google Cloud Java Client
+==========================
+
+Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
+
+[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
+[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master)
+
+-  [Homepage] (https://googlecloudplatform.github.io/gcloud-java/)
+-  [API Documentation] (http://googlecloudplatform.github.io/gcloud-java/apidocs)
+-  [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/examples/package-summary.html)
+
+This moudle provides common functionality and is required by the other service specific modules.
+
+Quickstart
+----------
+Add this to your pom.xml file
+```xml
+
+  com.google.gcloud
+  gcloud-java-core
+  LATEST
+
+```
+
+Contributing
+------------
+
+Contributions to this library are always welcome and highly encouraged.
+
+See [CONTRIBUTING] for more information on how to get started.
+
+Java Versions
+-------------
+
+Java 7 or above is required for using this client.
+
+Versioning
+----------
+
+This library follows [Semantic Versioning] (http://semver.org/).
+
+It is currently in major version zero (``0.y.z``), which means that anything
+may change at any time and the public API should not be considered
+stable.
+
+License
+-------
+
+Apache 2.0 - See [LICENSE] for more information.
+
+
+[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
+[cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml
new file mode 100644
index 000000000000..7fbbe7dad21d
--- /dev/null
+++ b/gcloud-java-core/pom.xml
@@ -0,0 +1,94 @@
+
+
+  4.0.0
+  com.google.gcloud
+  gcloud-java-core
+  jar
+  GCloud Java core
+  https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    Java idiomatic client for Google Cloud Platform services.
+  
+  
+    com.google.gcloud
+    gcloud-java-pom
+    0.0.5
+  
+  
+    
+      com.google.auth
+      google-auth-library-credentials
+      0.1.0
+    
+    
+      com.google.auth
+      google-auth-library-oauth2-http
+      0.1.0
+    
+    
+      com.google.http-client
+      google-http-client
+      1.19.0
+      compile
+    
+    
+      com.google.oauth-client
+      google-oauth-client
+      1.19.0
+      compile
+    
+    
+      com.google.guava
+      guava
+      18.0
+    
+    
+      com.google.api-client
+      google-api-client-appengine
+      1.20.0
+      compile
+      
+        
+          guava-jdk5
+          com.google.guava
+        
+      
+    
+    
+      com.google.http-client
+      google-http-client-jackson
+      1.20.0
+      compile
+      
+        
+          guava-jdk5
+          com.google.guava
+        
+      
+    
+    
+      junit
+      junit
+      4.12
+      test
+    
+    
+      joda-time
+      joda-time
+      RELEASE
+      compile
+    
+    
+      org.json
+      json
+      20090211
+      compile
+    
+    
+      org.easymock
+      easymock
+      3.3
+      test
+    
+  
+
diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
similarity index 100%
rename from src/main/java/com/google/gcloud/AuthCredentials.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
diff --git a/src/main/java/com/google/gcloud/BaseService.java b/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
similarity index 100%
rename from src/main/java/com/google/gcloud/BaseService.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
diff --git a/src/main/java/com/google/gcloud/ExceptionHandler.java b/gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java
similarity index 100%
rename from src/main/java/com/google/gcloud/ExceptionHandler.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/ExceptionHandler.java
diff --git a/src/main/java/com/google/gcloud/RetryHelper.java b/gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java
similarity index 100%
rename from src/main/java/com/google/gcloud/RetryHelper.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/RetryHelper.java
diff --git a/src/main/java/com/google/gcloud/RetryParams.java b/gcloud-java-core/src/main/java/com/google/gcloud/RetryParams.java
similarity index 100%
rename from src/main/java/com/google/gcloud/RetryParams.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/RetryParams.java
diff --git a/src/main/java/com/google/gcloud/Service.java b/gcloud-java-core/src/main/java/com/google/gcloud/Service.java
similarity index 100%
rename from src/main/java/com/google/gcloud/Service.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/Service.java
diff --git a/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
similarity index 100%
rename from src/main/java/com/google/gcloud/ServiceOptions.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
diff --git a/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
similarity index 100%
rename from src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
rename to gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
new file mode 100644
index 000000000000..41a2bf2d5848
--- /dev/null
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.spi;
+
+import com.google.common.collect.Iterables;
+import com.google.gcloud.ServiceOptions;
+
+import java.util.ServiceLoader;
+
+public class ServiceRpcProvider {
+
+  public static > R get(O options,
+      Class> factoryClass) {
+    ServiceRpcFactory factory = Iterables.getFirst(ServiceLoader.load(factoryClass), null);
+    return factory == null ? null : factory.create(options);
+  }
+}
+
diff --git a/src/test/java/com/google/gcloud/ExceptionHandlerTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/ExceptionHandlerTest.java
rename to gcloud-java-core/src/test/java/com/google/gcloud/ExceptionHandlerTest.java
diff --git a/src/test/java/com/google/gcloud/RetryHelperTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/RetryHelperTest.java
rename to gcloud-java-core/src/test/java/com/google/gcloud/RetryHelperTest.java
diff --git a/src/test/java/com/google/gcloud/RetryParamsTest.java b/gcloud-java-core/src/test/java/com/google/gcloud/RetryParamsTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/RetryParamsTest.java
rename to gcloud-java-core/src/test/java/com/google/gcloud/RetryParamsTest.java
diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
new file mode 100644
index 000000000000..96d8075706e9
--- /dev/null
+++ b/gcloud-java-datastore/README.md
@@ -0,0 +1,106 @@
+Google Cloud Java Client
+==========================
+
+Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
+
+[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
+[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master)
+
+-  [Homepage] (https://googlecloudplatform.github.io/gcloud-java/)
+-  [API Documentation] (http://googlecloudplatform.github.io/gcloud-java/apidocs)
+-  [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/examples/package-summary.html)
+
+This client supports [Google Cloud Datastore] (https://cloud.google.com/datastore/)
+
+
+> Note: This client is a work-in-progress, and may occasionally
+> make backwards-incompatible changes.
+
+Quickstart
+----------
+Add this to your pom.xml file
+```xml
+
+  com.google.gcloud
+  gcloud-java-datastore
+  LATEST
+
+```
+
+Google [Cloud Datastore][cloud-datastore] is a fully managed, schemaless database for
+storing non-relational data. Cloud Datastore automatically scales with
+your users and supports ACID transactions, high availability of reads and
+writes, strong consistency for reads and ancestor queries, and eventual
+consistency for all other queries.
+
+See the [Google Cloud Datastore docs][cloud-datastore-activation] for more details on how to activate
+Cloud Datastore for your project.
+
+See the ``gcloud-java`` API [datastore documentation][datastore-api] to learn how to interact
+with the Cloud Datastore using this Client Library.
+
+```java
+import com.google.gcloud.datastore.DatastoreService;
+import com.google.gcloud.datastore.DatastoreServiceFactory;
+import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.DateTime;
+import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.Key;
+import com.google.gcloud.datastore.KeyFactory;
+
+DatastoreServiceOptions options = DatastoreServiceOptions.builder().projectId(PROJECT_ID).build();
+DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
+KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
+Key key = keyFactory.newKey(keyName);
+Entity entity = datastore.get(key);
+if (entity == null) {
+  entity = Entity.builder(key)
+      .set("name", "John Do")
+      .set("age", 30)
+      .set("access_time", DateTime.now())
+      .build();
+  datastore.put(entity);
+} else {
+  System.out.println("Updating access_time for " + entity.getString("name"));
+  entity = Entity.builder(entity)
+      .set("access_time", DateTime.now())
+      .build();
+  datastore.update(entity);
+}
+```
+
+Contributing
+------------
+
+Contributions to this library are always welcome and highly encouraged.
+
+See [CONTRIBUTING] for more information on how to get started.
+
+Java Versions
+-------------
+
+Java 7 or above is required for using this client.
+
+Versioning
+----------
+
+This library follows [Semantic Versioning] (http://semver.org/).
+
+It is currently in major version zero (``0.y.z``), which means that anything
+may change at any time and the public API should not be considered
+stable.
+
+License
+-------
+
+Apache 2.0 - See [LICENSE] for more information.
+
+
+[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
+[cloud-platform]: https://cloud.google.com/
+[cloud-datastore]: https://cloud.google.com/datastore/docs
+[cloud-datastore-docs]: https://cloud.google.com/datastore/docs
+[cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate
+[datastore-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/datastore/package-summary.html
+
diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml
new file mode 100644
index 000000000000..0217e9cad58c
--- /dev/null
+++ b/gcloud-java-datastore/pom.xml
@@ -0,0 +1,44 @@
+
+
+  4.0.0
+  com.google.gcloud
+  gcloud-java-datastore
+  jar
+  GCloud Java datastore
+  https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    com.google.gcloud
+    gcloud-java-pom
+    0.0.5
+  
+  
+    
+      ${project.groupId}
+      gcloud-java-core
+      ${project.version}
+    
+    
+      com.google.apis
+      google-api-services-datastore-protobuf
+      v1beta2-rev1-2.1.2
+      compile
+    
+    
+      com.google.apis
+      google-api-services-datastore
+      v1beta2-rev23-1.19.0
+    
+    
+      junit
+      junit
+      4.12
+      test
+    
+    
+      org.easymock
+      easymock
+      3.3
+      test
+    
+  
+
diff --git a/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
diff --git a/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/BaseEntity.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
diff --git a/src/main/java/com/google/gcloud/datastore/BaseKey.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseKey.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/BaseKey.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseKey.java
diff --git a/src/main/java/com/google/gcloud/datastore/Batch.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Batch.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Batch.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Batch.java
diff --git a/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/BatchImpl.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
diff --git a/src/main/java/com/google/gcloud/datastore/BatchOption.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchOption.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/BatchOption.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchOption.java
diff --git a/src/main/java/com/google/gcloud/datastore/Blob.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Blob.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Blob.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Blob.java
diff --git a/src/main/java/com/google/gcloud/datastore/BlobValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BlobValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/BlobValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BlobValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/BooleanValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BooleanValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/BooleanValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BooleanValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/Cursor.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Cursor.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Cursor.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreReader.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReaderWriter.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreService.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreService.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreService.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
similarity index 95%
rename from src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index f7b1965d0b29..1dc0855baded 100644
--- a/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -26,6 +26,8 @@
 import com.google.gcloud.ServiceOptions;
 import com.google.gcloud.spi.DatastoreRpc;
 import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
+import com.google.gcloud.spi.DatastoreRpcFactory;
+import com.google.gcloud.spi.DefaultDatastoreRpc;
 import com.google.gcloud.spi.ServiceRpcProvider;
 
 import java.lang.reflect.Method;
@@ -185,7 +187,10 @@ DatastoreRpc datastoreRpc() {
     if (serviceRpcFactory() != null) {
       datastoreRpc = serviceRpcFactory().create(this);
     } else {
-      datastoreRpc = ServiceRpcProvider.datastore(this);
+      datastoreRpc = ServiceRpcProvider.get(this, DatastoreRpcFactory.class);
+      if (datastoreRpc == null) {
+        datastoreRpc = new DefaultDatastoreRpc(this);
+      }
     }
     return datastoreRpc;
   }
diff --git a/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
diff --git a/src/main/java/com/google/gcloud/datastore/DateTime.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DateTime.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java
diff --git a/src/main/java/com/google/gcloud/datastore/DateTimeValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DateTimeValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTimeValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/DoubleValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DoubleValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/DoubleValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DoubleValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/Entity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Entity.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java
diff --git a/src/main/java/com/google/gcloud/datastore/EntityValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/EntityValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/FullEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/FullEntity.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
diff --git a/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/GqlQuery.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java
diff --git a/src/main/java/com/google/gcloud/datastore/IncompleteKey.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/IncompleteKey.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/IncompleteKey.java
diff --git a/src/main/java/com/google/gcloud/datastore/Key.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Key.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Key.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Key.java
diff --git a/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/KeyFactory.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
diff --git a/src/main/java/com/google/gcloud/datastore/KeyValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/KeyValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/ListValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ListValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/ListValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ListValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/LongValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/LongValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/LongValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/LongValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/NullValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/NullValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/NullValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/NullValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/PathElement.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/PathElement.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/PathElement.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/PathElement.java
diff --git a/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
diff --git a/src/main/java/com/google/gcloud/datastore/Query.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Query.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResults.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/QueryResults.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
diff --git a/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
diff --git a/src/main/java/com/google/gcloud/datastore/RawValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/RawValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/RawValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/RawValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/Serializable.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Serializable.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Serializable.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Serializable.java
diff --git a/src/main/java/com/google/gcloud/datastore/StringValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StringValue.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/StringValue.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StringValue.java
diff --git a/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/StructuredQuery.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
diff --git a/src/main/java/com/google/gcloud/datastore/Transaction.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Transaction.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Transaction.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Transaction.java
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/TransactionImpl.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
diff --git a/src/main/java/com/google/gcloud/datastore/TransactionOption.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionOption.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/TransactionOption.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionOption.java
diff --git a/src/main/java/com/google/gcloud/datastore/Validator.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Validator.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Validator.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Validator.java
diff --git a/src/main/java/com/google/gcloud/datastore/Value.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Value.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/Value.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Value.java
diff --git a/src/main/java/com/google/gcloud/datastore/ValueBuilder.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueBuilder.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/ValueBuilder.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueBuilder.java
diff --git a/src/main/java/com/google/gcloud/datastore/ValueMarshaller.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueMarshaller.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/ValueMarshaller.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueMarshaller.java
diff --git a/src/main/java/com/google/gcloud/datastore/ValueType.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueType.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/ValueType.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueType.java
diff --git a/src/main/java/com/google/gcloud/datastore/package-info.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java
similarity index 100%
rename from src/main/java/com/google/gcloud/datastore/package-info.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java
diff --git a/src/main/java/com/google/gcloud/spi/DatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
similarity index 100%
rename from src/main/java/com/google/gcloud/spi/DatastoreRpc.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
new file mode 100644
index 000000000000..de9e87e4fa2e
--- /dev/null
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.spi;
+
+import com.google.gcloud.datastore.DatastoreServiceOptions;
+
+public interface DatastoreRpcFactory extends
+    ServiceRpcFactory {
+}
+
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
new file mode 100644
index 000000000000..3d909d368bc0
--- /dev/null
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.spi;
+
+import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
+import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
+import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
+import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
+import com.google.api.services.datastore.DatastoreV1.CommitRequest;
+import com.google.api.services.datastore.DatastoreV1.CommitResponse;
+import com.google.api.services.datastore.DatastoreV1.LookupRequest;
+import com.google.api.services.datastore.DatastoreV1.LookupResponse;
+import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
+import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
+import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
+import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
+import com.google.api.services.datastore.client.Datastore;
+import com.google.api.services.datastore.client.DatastoreException;
+import com.google.api.services.datastore.client.DatastoreFactory;
+import com.google.api.services.datastore.client.DatastoreOptions.Builder;
+import com.google.common.collect.ImmutableMap;
+import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DefaultDatastoreRpc implements DatastoreRpc {
+
+  private final Datastore client;
+
+  private static final ImmutableMap STR_TO_REASON;
+  private static final ImmutableMap HTTP_STATUS_TO_REASON;
+
+  static {
+    ImmutableMap.Builder builder = ImmutableMap.builder();
+    Map httpCodes = new HashMap<>();
+    for (Reason reason : Reason.values()) {
+      builder.put(reason.name(), reason);
+      httpCodes.put(reason.httpStatus(), reason);
+    }
+    STR_TO_REASON = builder.build();
+    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
+  }
+
+  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
+    client = DatastoreFactory.get().create(
+        new Builder()
+            .dataset(options.projectId())
+            .host(options.host())
+            .initializer(options.httpRequestInitializer())
+            .build());
+  }
+
+  private static DatastoreRpcException translate(DatastoreException exception) {
+    String message = exception.getMessage();
+    String reasonStr = "";
+    if (message != null) {
+      try {
+        JSONObject json = new JSONObject(new JSONTokener(message));
+        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
+        reasonStr = error.getString("reason");
+        message = error.getString("message");
+      } catch (JSONException ignore) {
+        // ignore - will be converted to unknown
+      }
+    }
+    Reason reason = STR_TO_REASON.get(reasonStr);
+    if (reason == null) {
+      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
+    }
+    return reason != null
+        ? new DatastoreRpcException(reason)
+        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
+  }
+
+  @Override
+  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
+      throws DatastoreRpcException {
+    try {
+      return client.allocateIds(request);
+    } catch (DatastoreException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
+      throws DatastoreRpcException {
+    try {
+      return client.beginTransaction(request);
+    } catch (DatastoreException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
+    try {
+      return client.commit(request);
+    } catch (DatastoreException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
+    try {
+      return client.lookup(request);
+    } catch (DatastoreException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
+    try {
+      return client.rollback(request);
+    } catch (DatastoreException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
+    try {
+      return client.runQuery(request);
+    } catch (DatastoreException ex) {
+      throw translate(ex);
+    }
+  }
+}
+
diff --git a/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseKeyTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/BlobTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/BlobTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/BlobValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/BlobValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BooleanValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/CursorTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/CursorTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/DateTimeTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/DateTimeTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DoubleValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/EntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/EntityTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/EntityValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/EntityValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/FullEntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/FullEntityTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/FullEntityTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/FullEntityTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/IncompleteKeyTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyFactoryTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/KeyTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/KeyTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/KeyValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/KeyValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/KeyValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/ListValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ListValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/ListValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ListValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
diff --git a/src/test/java/com/google/gcloud/datastore/LongValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LongValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/LongValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LongValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/NullValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/NullValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/NullValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/NullValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/PathElementTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/PathElementTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/PathElementTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/PathElementTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/RawValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/RawValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/RawValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/RawValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/SerializationTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/StringValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/StringValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/StringValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/StringValueTest.java
diff --git a/src/test/java/com/google/gcloud/datastore/ValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/datastore/ValueTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md
new file mode 100644
index 000000000000..59df6ced388f
--- /dev/null
+++ b/gcloud-java-examples/README.md
@@ -0,0 +1,55 @@
+Google Cloud Java Client Examples
+=================================
+
+Examples for gcloud-java (Java idiomatic client for [Google Cloud Platform][cloud-platform] services).
+
+[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
+[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master)
+
+-  [Homepage] (https://googlecloudplatform.github.io/gcloud-java/)
+-  [API Documentation] (http://googlecloudplatform.github.io/gcloud-java/apidocs)
+-  [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/examples/package-summary.html)
+
+
+Quickstart
+----------
+Add this to your pom.xml file
+```xml
+
+  com.google.gcloud
+  gcloud-java-examples
+  LATEST
+
+```
+
+
+Contributing
+------------
+
+Contributions to this library are always welcome and highly encouraged.
+
+See [CONTRIBUTING] for more information on how to get started.
+
+Java Versions
+-------------
+
+Java 7 or above is required for using this client.
+
+Versioning
+----------
+
+This library follows [Semantic Versioning] (http://semver.org/).
+
+It is currently in major version zero (``0.y.z``), which means that anything
+may change at any time and the public API should not be considered
+stable.
+
+License
+-------
+
+Apache 2.0 - See [LICENSE] for more information.
+
+
+[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
+[cloud-platform]: https://cloud.google.com/
diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml
new file mode 100644
index 000000000000..ac1ef9f667cb
--- /dev/null
+++ b/gcloud-java-examples/pom.xml
@@ -0,0 +1,37 @@
+
+
+  4.0.0
+  com.google.gcloud
+  gcloud-java-examples
+  jar
+  GCloud Java examples
+  https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    Examples for gcloud-java.
+  
+  
+    com.google.gcloud
+    gcloud-java-pom
+    0.0.5
+  
+  
+    
+      ${project.groupId}
+      gcloud-java
+      ${project.version}
+    
+  
+  
+    
+      
+        org.apache.maven.plugins
+        maven-site-plugin
+        3.1
+        
+          true
+          true
+        
+      
+    
+  
+
diff --git a/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
similarity index 100%
rename from src/main/java/com/google/gcloud/examples/DatastoreExample.java
rename to gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
similarity index 100%
rename from src/main/java/com/google/gcloud/examples/StorageExample.java
rename to gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
diff --git a/gcloud-java-storage/README.md b/gcloud-java-storage/README.md
new file mode 100644
index 000000000000..aa090743f5ce
--- /dev/null
+++ b/gcloud-java-storage/README.md
@@ -0,0 +1,63 @@
+Google Cloud Java Client
+==========================
+
+Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
+
+[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
+[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master)
+
+-  [Homepage] (https://googlecloudplatform.github.io/gcloud-java/)
+-  [API Documentation] (http://googlecloudplatform.github.io/gcloud-java/apidocs)
+-  [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/examples/package-summary.html)
+
+This client supports the [Google Cloud Storage] (https://cloud.google.com/storage/)
+
+> Note: This client is a work-in-progress, and may occasionally
+> make backwards-incompatible changes.
+
+Quickstart
+----------
+Add this to your pom.xml file
+```xml
+
+  com.google.gcloud
+  gcloud-java-storage
+  LATEST
+
+```
+
+
+Contributing
+------------
+
+Contributions to this library are always welcome and highly encouraged.
+
+See [CONTRIBUTING] for more information on how to get started.
+
+Java Versions
+-------------
+
+Java 7 or above is required for using this client.
+
+Versioning
+----------
+
+This library follows [Semantic Versioning] (http://semver.org/).
+
+It is currently in major version zero (``0.y.z``), which means that anything
+may change at any time and the public API should not be considered
+stable.
+
+License
+-------
+
+Apache 2.0 - See [LICENSE] for more information.
+
+
+[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
+[cloud-platform]: https://cloud.google.com/
+
+[cloud-storage]: https://cloud.google.com/storage/
+[cloud-storage-docs]: https://cloud.google.com/storage/docs/overview
+[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml
new file mode 100644
index 000000000000..29731998eeca
--- /dev/null
+++ b/gcloud-java-storage/pom.xml
@@ -0,0 +1,39 @@
+
+
+  4.0.0
+  com.google.gcloud
+  gcloud-java-storage
+  jar
+  GCloud Java storage
+  https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    com.google.gcloud
+    gcloud-java-pom
+    0.0.5
+  
+  
+    
+      ${project.groupId}
+      gcloud-java-core
+      ${project.version}
+    
+    
+      com.google.apis
+      google-api-services-storage
+      v1-rev33-1.20.0
+      compile
+    
+    
+      junit
+      junit
+      4.12
+      test
+    
+    
+      org.easymock
+      easymock
+      3.3
+      test
+    
+  
+
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
new file mode 100644
index 000000000000..e27d837d7173
--- /dev/null
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -0,0 +1,525 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.gcloud.spi;
+
+import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
+import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
+import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
+import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
+import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
+import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
+import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;
+
+import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
+import com.google.api.client.googleapis.json.GoogleJsonError;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.googleapis.media.MediaHttpDownloader;
+import com.google.api.client.http.ByteArrayContent;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.json.JsonHttpContent;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.Storage.Objects.Get;
+import com.google.api.services.storage.Storage.Objects.Insert;
+import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.Buckets;
+import com.google.api.services.storage.model.ComposeRequest;
+import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
+import com.google.api.services.storage.model.Objects;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.gcloud.storage.StorageServiceException;
+import com.google.gcloud.storage.StorageServiceOptions;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class DefaultStorageRpc implements StorageRpc {
+
+  public static final String DEFAULT_PROJECTION = "full";
+  private final StorageServiceOptions options;
+  private final Storage storage;
+
+  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
+  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);
+
+  public DefaultStorageRpc(StorageServiceOptions options) {
+    HttpTransport transport = options.httpTransportFactory().create();
+    HttpRequestInitializer initializer = options.httpRequestInitializer();
+    this.options = options;
+    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
+        .setApplicationName("gcloud-java")
+        .build();
+    // Todo: make sure nulls are being used as Data.asNull()
+  }
+
+  private static StorageServiceException translate(IOException exception) {
+    StorageServiceException translated;
+    if (exception instanceof GoogleJsonResponseException) {
+      translated = translate(((GoogleJsonResponseException) exception).getDetails());
+    } else {
+      translated = new StorageServiceException(0, exception.getMessage(), false);
+    }
+    translated.initCause(exception);
+    return translated;
+  }
+
+  private static StorageServiceException translate(GoogleJsonError exception) {
+    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
+        || "InternalError".equals(exception.getMessage());
+    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
+  }
+
+  @Override
+  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
+    try {
+      return storage.buckets()
+          .insert(this.options.projectId(), bucket)
+          .setProjection(DEFAULT_PROJECTION)
+          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
+          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
+          .execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public StorageObject create(StorageObject storageObject, final byte[] content,
+      Map options) throws StorageServiceException {
+    try {
+      return storage.objects()
+          .insert(storageObject.getBucket(), storageObject,
+              new ByteArrayContent(storageObject.getContentType(), content))
+          .setProjection(DEFAULT_PROJECTION)
+          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
+          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
+          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
+          .execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public Tuple> list(Map options) {
+    try {
+      Buckets buckets = storage.buckets()
+          .list(this.options.projectId())
+          .setProjection(DEFAULT_PROJECTION)
+          .setPrefix(PREFIX.getString(options))
+          .setMaxResults(MAX_RESULTS.getLong(options))
+          .setPageToken(PAGE_TOKEN.getString(options))
+          .execute();
+      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public Tuple> list(String bucket, Map options) {
+    try {
+      Objects objects = storage.objects()
+          .list(bucket)
+          .setProjection(DEFAULT_PROJECTION)
+          .setVersions(VERSIONS.getBoolean(options))
+          .setDelimiter(DELIMITER.getString(options))
+          .setPrefix(PREFIX.getString(options))
+          .setMaxResults(MAX_RESULTS.getLong(options))
+          .setPageToken(PAGE_TOKEN.getString(options))
+          .execute();
+      return Tuple.>of(
+          objects.getNextPageToken(), objects.getItems());
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public Bucket get(Bucket bucket, Map options) {
+    try {
+      return storage.buckets()
+          .get(bucket.getName())
+          .setProjection(DEFAULT_PROJECTION)
+          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+          .execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public StorageObject get(StorageObject object, Map options) {
+    try {
+      return getRequest(object, options).execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  private Storage.Objects.Get getRequest(StorageObject object, Map options)
+      throws IOException {
+    return storage.objects()
+        .get(object.getBucket(), object.getName())
+        .setProjection(DEFAULT_PROJECTION)
+        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
+        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
+  }
+
+  @Override
+  public Bucket patch(Bucket bucket, Map options) {
+    try {
+      return storage.buckets()
+          .patch(bucket.getName(), bucket)
+          .setProjection(DEFAULT_PROJECTION)
+          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
+          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
+          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+          .execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public StorageObject patch(StorageObject storageObject, Map options) {
+    try {
+      return patchRequest(storageObject, options).execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
+      throws IOException {
+    return storage.objects()
+        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
+        .setProjection(DEFAULT_PROJECTION)
+        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
+        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
+        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
+  }
+
+  @Override
+  public boolean delete(Bucket bucket, Map options) {
+    try {
+      storage.buckets()
+          .delete(bucket.getName())
+          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+          .execute();
+      return true;
+    } catch (IOException ex) {
+      StorageServiceException serviceException = translate(ex);
+      if (serviceException.code() == 404) {
+        return false;
+      }
+      throw serviceException;
+    }
+  }
+
+  @Override
+  public boolean delete(StorageObject blob, Map options) {
+    try {
+      deleteRequest(blob, options).execute();
+      return true;
+    } catch (IOException ex) {
+      StorageServiceException serviceException = translate(ex);
+      if (serviceException.code() == 404) {
+        return false;
+      }
+      throw serviceException;
+    }
+  }
+
+  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
+      throws IOException {
+    return storage.objects()
+        .delete(blob.getBucket(), blob.getName())
+        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
+        .setIfGenerationMatch(100L)
+        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
+  }
+
+  @Override
+  public StorageObject compose(Iterable sources, StorageObject target,
+      Map targetOptions) throws StorageServiceException {
+    ComposeRequest request = new ComposeRequest();
+    if (target.getContentType() == null) {
+      // todo: remove once this is no longer requirement (b/20681287).
+      target.setContentType("application/octet-stream");
+    }
+    request.setDestination(target);
+    List sourceObjects = new ArrayList<>();
+    for (StorageObject source : sources) {
+      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
+      sourceObject.setName(source.getName());
+      Long generation = source.getGeneration();
+      if (generation != null) {
+        sourceObject.setGeneration(generation);
+        sourceObject.setObjectPreconditions(
+            new ObjectPreconditions().setIfGenerationMatch(generation));
+      }
+      sourceObjects.add(sourceObject);
+    }
+    request.setSourceObjects(sourceObjects);
+    try {
+      // todo: missing setProjection (b/20659000)
+      return storage.objects()
+          .compose(target.getBucket(), target.getName(), request)
+          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
+          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
+          .execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public StorageObject copy(StorageObject source, Map sourceOptions,
+      StorageObject target, Map targetOptions) throws StorageServiceException {
+    try {
+      return storage
+          .objects()
+          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
+              target.getContentType() != null ? target : null)
+          .setProjection(DEFAULT_PROJECTION)
+          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
+          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
+          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
+          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
+          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
+          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
+          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
+          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
+          .execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public byte[] load(StorageObject from, Map options)
+      throws StorageServiceException {
+    try {
+      Storage.Objects.Get getRequest = storage.objects()
+          .get(from.getBucket(), from.getName())
+          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
+          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
+      getRequest.executeMediaAndDownloadTo(out);
+      return out.toByteArray();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
+    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
+    final Map> deletes =
+        Maps.newConcurrentMap();
+    final Map> updates =
+        Maps.newConcurrentMap();
+    final Map> gets =
+        Maps.newConcurrentMap();
+    try {
+      for (final Tuple> tuple : request.toDelete) {
+        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
+          @Override
+          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
+            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
+          }
+
+          @Override
+          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
+            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
+          }
+        });
+      }
+      for (final Tuple> tuple : request.toUpdate) {
+        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
+          @Override
+          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
+            updates.put(tuple.x(),
+                Tuple.of(storageObject, null));
+          }
+
+          @Override
+          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
+            updates.put(tuple.x(),
+                Tuple.of(null, translate(e)));
+          }
+        });
+      }
+      for (final Tuple> tuple : request.toGet) {
+        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
+          @Override
+          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
+            gets.put(tuple.x(),
+                Tuple.of(storageObject, null));
+          }
+
+          @Override
+          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
+            gets.put(tuple.x(),
+                Tuple.of(null, translate(e)));
+          }
+        });
+      }
+      batch.execute();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+    return new BatchResponse(deletes, updates, gets);
+  }
+
+  @Override
+  public byte[] read(StorageObject from, Map options, long position, int bytes)
+      throws StorageServiceException {
+    try {
+      Get req = storage.objects().get(from.getBucket(), from.getName());
+      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
+          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
+          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
+          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
+      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
+      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
+      downloader.setContentRange(position, (int) position + bytes);
+      downloader.setDirectDownloadEnabled(true);
+      ByteArrayOutputStream output = new ByteArrayOutputStream();
+      req.executeMediaAndDownloadTo(output);
+      return output.toByteArray();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
+      long destOffset, int length, boolean last) throws StorageServiceException {
+    try {
+      GenericUrl url = new GenericUrl(uploadId);
+      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
+          new ByteArrayContent(null, toWrite, toWriteOffset, length));
+      long limit = destOffset + length;
+      StringBuilder range = new StringBuilder("bytes ");
+      range.append(destOffset).append('-').append(limit - 1).append('/');
+      if (last) {
+        range.append(limit);
+      } else {
+        range.append('*');
+      }
+      httpRequest.getHeaders().setContentRange(range.toString());
+      int code;
+      String message;
+      IOException exception = null;
+      try {
+        HttpResponse response = httpRequest.execute();
+        code = response.getStatusCode();
+        message = response.getStatusMessage();
+      } catch (HttpResponseException ex) {
+        exception = ex;
+        code = ex.getStatusCode();
+        message = ex.getStatusMessage();
+      }
+      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
+        if (exception != null) {
+          throw exception;
+        }
+        GoogleJsonError error = new GoogleJsonError();
+        error.setCode(code);
+        error.setMessage(message);
+        throw translate(error);
+      }
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+
+  @Override
+  public String open(StorageObject object, Map options)
+      throws StorageServiceException {
+    try {
+      Insert req = storage.objects().insert(object.getBucket(), object);
+      GenericUrl url = req.buildHttpRequest().getUrl();
+      String scheme = url.getScheme();
+      String host = url.getHost();
+      String path = "/upload" + url.getRawPath();
+      url = new GenericUrl(scheme + "://" + host + path);
+      url.set("uploadType", "resumable");
+      url.set("name", object.getName());
+      for (Option option : options.keySet()) {
+        Object content = option.get(options);
+        if (content != null) {
+          url.set(option.value(), content.toString());
+        }
+      }
+      JsonFactory jsonFactory = storage.getJsonFactory();
+      HttpRequestFactory requestFactory = storage.getRequestFactory();
+      HttpRequest httpRequest =
+          requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
+      httpRequest.getHeaders().set("X-Upload-Content-Type",
+          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
+      HttpResponse response = httpRequest.execute();
+      if (response.getStatusCode() != 200) {
+        GoogleJsonError error = new GoogleJsonError();
+        error.setCode(response.getStatusCode());
+        error.setMessage(response.getStatusMessage());
+        throw translate(error);
+      }
+      return response.getHeaders().getLocation();
+    } catch (IOException ex) {
+      throw translate(ex);
+    }
+  }
+}
+
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
similarity index 100%
rename from src/main/java/com/google/gcloud/spi/StorageRpc.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
new file mode 100644
index 000000000000..d78e721e2331
--- /dev/null
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.spi;
+
+import com.google.gcloud.storage.StorageServiceOptions;
+
+public interface StorageRpcFactory extends ServiceRpcFactory {
+}
+
diff --git a/src/main/java/com/google/gcloud/storage/Acl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/Acl.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java
diff --git a/src/main/java/com/google/gcloud/storage/BatchRequest.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/BatchRequest.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
diff --git a/src/main/java/com/google/gcloud/storage/BatchResponse.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/BatchResponse.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
diff --git a/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/Blob.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/BlobReadChannel.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
diff --git a/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/Bucket.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Cors.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/Cors.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/Cors.java
diff --git a/src/main/java/com/google/gcloud/storage/ListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/ListResult.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java
diff --git a/src/main/java/com/google/gcloud/storage/Option.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Option.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/Option.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/Option.java
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/StorageService.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceException.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/StorageServiceException.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceException.java
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
similarity index 92%
rename from src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index 87cac636fa92..a458692511bd 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -19,8 +19,10 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableSet;
 import com.google.gcloud.ServiceOptions;
+import com.google.gcloud.spi.DefaultStorageRpc;
 import com.google.gcloud.spi.ServiceRpcProvider;
 import com.google.gcloud.spi.StorageRpc;
+import com.google.gcloud.spi.StorageRpcFactory;
 
 import java.util.Objects;
 import java.util.Set;
@@ -75,7 +77,10 @@ StorageRpc storageRpc() {
     if (serviceRpcFactory() != null) {
       storageRpc = serviceRpcFactory().create(this);
     } else {
-      storageRpc = ServiceRpcProvider.storage(this);
+      storageRpc = ServiceRpcProvider.get(this, StorageRpcFactory.class);
+      if (storageRpc == null) {
+        storageRpc = new DefaultStorageRpc(this);
+      }
     }
     return storageRpc;
   }
diff --git a/src/main/java/com/google/gcloud/storage/package-info.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
similarity index 100%
rename from src/main/java/com/google/gcloud/storage/package-info.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
diff --git a/src/test/java/com/google/gcloud/storage/AclTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/AclTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/AclTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/AclTest.java
diff --git a/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/BatchRequestTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
diff --git a/src/test/java/com/google/gcloud/storage/BatchResponseTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/BatchResponseTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
diff --git a/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/BlobTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
diff --git a/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/BucketTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java
diff --git a/src/test/java/com/google/gcloud/storage/CorsTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/CorsTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java
diff --git a/src/test/java/com/google/gcloud/storage/ListResultTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/ListResultTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java
diff --git a/src/test/java/com/google/gcloud/storage/OptionTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/OptionTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/OptionTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/OptionTest.java
diff --git a/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
similarity index 100%
rename from src/test/java/com/google/gcloud/storage/SerializationTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
diff --git a/gcloud-java/README.md b/gcloud-java/README.md
new file mode 100644
index 000000000000..2286627662fb
--- /dev/null
+++ b/gcloud-java/README.md
@@ -0,0 +1,73 @@
+Google Cloud Java Client
+==========================
+
+Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
+
+[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java)
+[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master)
+
+-  [Homepage] (https://googlecloudplatform.github.io/gcloud-java/)
+-  [API Documentation] (http://googlecloudplatform.github.io/gcloud-java/apidocs)
+-  [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/examples/package-summary.html)
+
+This client supports the following Google Cloud Platform services:
+
+-  [Google Cloud Datastore] (https://cloud.google.com/datastore/) [datastore documentation][datastore-api] 
+-  [Google Cloud Storage] (https://cloud.google.com/storage/) [storage documentation][storage-api] 
+
+> Note: This client is a work-in-progress, and may occasionally
+> make backwards-incompatible changes.
+
+Quickstart
+----------
+Add this to your pom.xml file
+```xml
+
+  com.google.gcloud
+  gcloud-java
+  LATEST
+
+```
+
+Contributing
+------------
+
+Contributions to this library are always welcome and highly encouraged.
+
+See [CONTRIBUTING] for more information on how to get started.
+
+Java Versions
+-------------
+
+Java 7 or above is required for using this client.
+
+Versioning
+----------
+
+This library follows [Semantic Versioning] (http://semver.org/).
+
+It is currently in major version zero (``0.y.z``), which means that anything
+may change at any time and the public API should not be considered
+stable.
+
+License
+-------
+
+Apache 2.0 - See [LICENSE] for more information.
+
+
+[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md
+[LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE
+[cloud-platform]: https://cloud.google.com/
+[cloud-datastore]: https://cloud.google.com/datastore/docs
+[cloud-datastore-docs]: https://cloud.google.com/datastore/docs
+[cloud-datastore-activation]: https://cloud.google.com/datastore/docs/activate
+[datastore-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/datastore/package-summary.html
+
+[cloud-pubsub]: https://cloud.google.com/pubsub/
+[cloud-pubsub-docs]: https://cloud.google.com/pubsub/docs
+
+[cloud-storage]: https://cloud.google.com/storage/
+[cloud-storage-docs]: https://cloud.google.com/storage/docs/overview
+[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets
+[storage-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/storage/package-summary.html
diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml
new file mode 100644
index 000000000000..a70763d2b1bd
--- /dev/null
+++ b/gcloud-java/pom.xml
@@ -0,0 +1,47 @@
+
+
+  4.0.0
+  com.google.gcloud
+  gcloud-java
+  jar
+  GCloud Java assembly
+  https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    Java idiomatic client for Google Cloud Platform services.
+  
+  
+    com.google.gcloud
+    gcloud-java-pom
+    0.0.5
+  
+  
+    
+      ${project.groupId}
+      gcloud-java-core
+      ${project.version}
+    
+    
+      ${project.groupId}
+      gcloud-java-datastore
+      ${project.version}
+    
+    
+      ${project.groupId}
+      gcloud-java-storage
+      ${project.version}
+    
+  
+  
+    
+      
+        org.apache.maven.plugins
+        maven-site-plugin
+        3.1
+        
+          true
+          true
+        
+      
+    
+  
+
diff --git a/pom.xml b/pom.xml
index db7388fb7ae1..a00e0d641b30 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2,9 +2,9 @@
 
   4.0.0
   com.google.gcloud
-  gcloud-java
-  jar
-  0.0.4
+  gcloud-java-pom
+  pom
+  0.0.5
   GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   
@@ -47,99 +47,12 @@
       sonatype-nexus-staging
       https://oss.sonatype.org/service/local/staging/deploy/maven2/
     
+    
+      github-pages-site
+      Deployment through GitHub's site deployment plugin
+      http://googlecloudplatform.github.io/gcloud-java/
+    
   
-  
-    
-      com.google.auth
-      google-auth-library-credentials
-      0.1.0
-    
-    
-      com.google.auth
-      google-auth-library-oauth2-http
-      0.1.0
-    
-    
-      com.google.http-client
-      google-http-client
-      1.19.0
-      compile
-    
-    
-      com.google.oauth-client
-      google-oauth-client
-      1.19.0
-      compile
-    
-    
-      com.google.guava
-      guava
-      18.0
-    
-    
-      com.google.apis
-      google-api-services-datastore-protobuf
-      v1beta2-rev1-2.1.2
-      compile
-    
-    
-      com.google.api-client
-      google-api-client-appengine
-      1.19.0
-      compile
-      
-        
-          guava-jdk5
-          com.google.guava
-        
-      
-    
-    
-      junit
-      junit
-      4.12
-      test
-    
-    
-      joda-time
-      joda-time
-      RELEASE
-      compile
-    
-    
-      org.json
-      json
-      20090211
-      compile
-    
-    
-      com.google.apis
-      google-api-services-storage
-      v1-rev33-1.20.0
-      compile
-    
-    
-      com.google.apis
-      google-api-services-datastore
-      v1beta2-rev23-1.19.0
-    
-    
-      org.easymock
-      easymock
-      3.3
-      test
-    
-  
-  
-    
-      
-        false
-      
-      central
-      Central Repository
-      http://repo.maven.apache.org/maven2
-    
-  
   
     
       GCloud Java Software License
@@ -151,6 +64,13 @@
     UTF-8
     github
   
+  
+    gcloud-java-core
+    gcloud-java-datastore
+    gcloud-java-storage
+    gcloud-java
+    gcloud-java-examples
+  
   
     
       
@@ -178,11 +98,10 @@
           
         
         
-          
+          
           -1
         
       
@@ -289,22 +208,22 @@
         org.codehaus.mojo
         cobertura-maven-plugin
         2.6
-        
-          
-            xml
-            html
-          
-          true
-          
-            true
-            
-              com/google/gcloud/**/*.class
-            
-            
-              com/google/gcloud/examples/**/*.class
-            
-          
-          
+          
+            
+              xml
+              html
+            
+            true
+            
+              true
+              
+                com/google/gcloud/**/*.class
+              
+              
+                com/google/gcloud/examples/**/*.class
+              
+            
+            
           256m
         
       
@@ -320,11 +239,12 @@
               
         
       
-      
+     
         org.apache.maven.plugins
         maven-site-plugin
         3.4
         
+          true
           
             
               org.apache.maven.plugins
@@ -338,6 +258,7 @@
               
                 
                   
+                    index
                     dependencies
                     project-team
                     mailing-list
@@ -351,11 +272,21 @@
                 
               
               
+                true
                 true
                 true
                 true
               
             
+            
+              org.apache.maven.plugins
+              maven-dependency-plugin
+              2.10
+              
+                true
+                ${project.build.directory}/dependencies
+              
+            
             
               org.apache.maven.plugins
               maven-javadoc-plugin
@@ -369,6 +300,7 @@
                 
               
               
+                true
                 protected
                 true
                 ${project.build.directory}/javadoc
@@ -392,6 +324,9 @@
               org.codehaus.mojo
               cobertura-maven-plugin
               2.6
+              
+                true
+              
             
           
         
@@ -402,34 +337,16 @@
         0.10
         
           Creating site for ${project.artifactId} ${project.version}
+          ${project.distributionManagement.site.url}
+          true
         
         
           
+            github-site
             
               site
             
-            site
-          
-        
-      
-      
-        org.apache.maven.plugins
-        maven-assembly-plugin
-        2.4.1
-        
-          
-          
-            jar-with-dependencies
-          
-        
-        
-          
-            make-assembly
-            
-            package
-            
-              single
-            
+            site-deploy
           
         
       
diff --git a/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java b/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
deleted file mode 100644
index c25a22aae717..000000000000
--- a/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.datastore.DatastoreServiceOptions;

public interface DatastoreRpcFactory extends
    ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
deleted file mode 100644
index 20bd911b3782..000000000000
--- a/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
import com.google.api.services.datastore.client.Datastore;
import com.google.api.services.datastore.client.DatastoreException;
import com.google.api.services.datastore.client.DatastoreFactory;
import com.google.api.services.datastore.client.DatastoreOptions.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;

import java.util.HashMap;
import java.util.Map;

class DefaultDatastoreRpc implements DatastoreRpc {

  private final Datastore client;

  private static final ImmutableMap STR_TO_REASON;
  private static final ImmutableMap HTTP_STATUS_TO_REASON;

  static {
    ImmutableMap.Builder builder = ImmutableMap.builder();
    Map httpCodes = new HashMap<>();
    for (Reason reason : Reason.values()) {
      builder.put(reason.name(), reason);
      httpCodes.put(reason.httpStatus(), reason);
    }
    STR_TO_REASON = builder.build();
    HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
  }

  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
    client = DatastoreFactory.get().create(
        new Builder()
            .dataset(options.projectId())
            .host(options.host())
            .initializer(options.httpRequestInitializer())
            .build());
  }

  private static DatastoreRpcException translate(DatastoreException exception) {
    String message = exception.getMessage();
    String reasonStr = "";
    if (message != null) {
      try {
        JSONObject json = new JSONObject(new JSONTokener(message));
        JSONObject error = json.getJSONObject("error").getJSONArray("errors").getJSONObject(0);
        reasonStr = error.getString("reason");
        message = error.getString("message");
      } catch (JSONException ignore) {
        // ignore - will be converted to unknown
      }
    }
    Reason reason = STR_TO_REASON.get(reasonStr);
    if (reason == null) {
      reason = HTTP_STATUS_TO_REASON.get(exception.getCode());
    }
    return reason != null
        ? new DatastoreRpcException(reason)
        : new DatastoreRpcException("Unknown", exception.getCode(), false, message);
  }

  @Override
  public AllocateIdsResponse allocateIds(AllocateIdsRequest request)
      throws DatastoreRpcException {
    try {
      return client.allocateIds(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException {
    try {
      return client.beginTransaction(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public CommitResponse commit(CommitRequest request) throws DatastoreRpcException {
    try {
      return client.commit(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public LookupResponse lookup(LookupRequest request) throws DatastoreRpcException {
    try {
      return client.lookup(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException {
    try {
      return client.rollback(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }

  @Override
  public RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException {
    try {
      return client.runQuery(request);
    } catch (DatastoreException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
deleted file mode 100644
index 455624f79b65..000000000000
--- a/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package com.google.gcloud.spi;

import static com.google.gcloud.spi.StorageRpc.Option.DELIMITER;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_GENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
import static com.google.gcloud.spi.StorageRpc.Option.MAX_RESULTS;
import static com.google.gcloud.spi.StorageRpc.Option.PAGE_TOKEN;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREDEFINED_DEFAULT_OBJECT_ACL;
import static com.google.gcloud.spi.StorageRpc.Option.PREFIX;
import static com.google.gcloud.spi.StorageRpc.Option.VERSIONS;

import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
import com.google.api.client.googleapis.json.GoogleJsonError;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.googleapis.media.MediaHttpDownloader;
import com.google.api.client.http.ByteArrayContent;
import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpHeaders;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseException;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.json.JsonHttpContent;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson.JacksonFactory;
import com.google.api.services.storage.Storage;
import com.google.api.services.storage.Storage.Objects.Get;
import com.google.api.services.storage.Storage.Objects.Insert;
import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.Buckets;
import com.google.api.services.storage.model.ComposeRequest;
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
import com.google.api.services.storage.model.Objects;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gcloud.storage.StorageServiceException;
import com.google.gcloud.storage.StorageServiceOptions;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class DefaultStorageRpc implements StorageRpc {

  public static final String DEFAULT_PROJECTION = "full";
  private final StorageServiceOptions options;
  private final Storage storage;

  // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
  private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);

  public DefaultStorageRpc(StorageServiceOptions options) {
    HttpTransport transport = options.httpTransportFactory().create();
    HttpRequestInitializer initializer = options.httpRequestInitializer();
    this.options = options;
    storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
        .setApplicationName("gcloud-java")
        .build();
    // Todo: make sure nulls are being used as Data.asNull()
  }

  private static StorageServiceException translate(IOException exception) {
    StorageServiceException translated;
    if (exception instanceof GoogleJsonResponseException) {
      translated = translate(((GoogleJsonResponseException) exception).getDetails());
    } else {
      translated = new StorageServiceException(0, exception.getMessage(), false);
    }
    translated.initCause(exception);
    return translated;
  }

  private static StorageServiceException translate(GoogleJsonError exception) {
    boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
        || "InternalError".equals(exception.getMessage());
    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
  }

  @Override
  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
    try {
      return storage.buckets()
          .insert(this.options.projectId(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject create(StorageObject storageObject, final byte[] content,
      Map options) throws StorageServiceException {
    try {
      return storage.objects()
          .insert(storageObject.getBucket(), storageObject,
              new ByteArrayContent(storageObject.getContentType(), content))
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(Map options) {
    try {
      Buckets buckets = storage.buckets()
          .list(this.options.projectId())
          .setProjection(DEFAULT_PROJECTION)
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(buckets.getNextPageToken(), buckets.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Tuple> list(String bucket, Map options) {
    try {
      Objects objects = storage.objects()
          .list(bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setVersions(VERSIONS.getBoolean(options))
          .setDelimiter(DELIMITER.getString(options))
          .setPrefix(PREFIX.getString(options))
          .setMaxResults(MAX_RESULTS.getLong(options))
          .setPageToken(PAGE_TOKEN.getString(options))
          .execute();
      return Tuple.>of(
          objects.getNextPageToken(), objects.getItems());
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public Bucket get(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .get(bucket.getName())
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject get(StorageObject object, Map options) {
    try {
      return getRequest(object, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Get getRequest(StorageObject object, Map options)
      throws IOException {
    return storage.objects()
        .get(object.getBucket(), object.getName())
        .setProjection(DEFAULT_PROJECTION)
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public Bucket patch(Bucket bucket, Map options) {
    try {
      return storage.buckets()
          .patch(bucket.getName(), bucket)
          .setProjection(DEFAULT_PROJECTION)
          .setPredefinedAcl(PREDEFINED_ACL.getString(options))
          .setPredefinedDefaultObjectAcl(PREDEFINED_DEFAULT_OBJECT_ACL.getString(options))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject patch(StorageObject storageObject, Map options) {
    try {
      return patchRequest(storageObject, options).execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  private Storage.Objects.Patch patchRequest(StorageObject storageObject, Map options)
      throws IOException {
    return storage.objects()
        .patch(storageObject.getBucket(), storageObject.getName(), storageObject)
        .setProjection(DEFAULT_PROJECTION)
        .setPredefinedAcl(PREDEFINED_ACL.getString(options))
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public boolean delete(Bucket bucket, Map options) {
    try {
      storage.buckets()
          .delete(bucket.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  @Override
  public boolean delete(StorageObject blob, Map options) {
    try {
      deleteRequest(blob, options).execute();
      return true;
    } catch (IOException ex) {
      StorageServiceException serviceException = translate(ex);
      if (serviceException.code() == 404) {
        return false;
      }
      throw serviceException;
    }
  }

  private Storage.Objects.Delete deleteRequest(StorageObject blob, Map options)
      throws IOException {
    return storage.objects()
        .delete(blob.getBucket(), blob.getName())
        .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
        .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
        .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
        .setIfGenerationMatch(100L)
        .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
  }

  @Override
  public StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException {
    ComposeRequest request = new ComposeRequest();
    if (target.getContentType() == null) {
      // todo: remove once this is no longer requirement (b/20681287).
      target.setContentType("application/octet-stream");
    }
    request.setDestination(target);
    List sourceObjects = new ArrayList<>();
    for (StorageObject source : sources) {
      ComposeRequest.SourceObjects sourceObject = new ComposeRequest.SourceObjects();
      sourceObject.setName(source.getName());
      Long generation = source.getGeneration();
      if (generation != null) {
        sourceObject.setGeneration(generation);
        sourceObject.setObjectPreconditions(
            new ObjectPreconditions().setIfGenerationMatch(generation));
      }
      sourceObjects.add(sourceObject);
    }
    request.setSourceObjects(sourceObjects);
    try {
      // todo: missing setProjection (b/20659000)
      return storage.objects()
          .compose(target.getBucket(), target.getName(), request)
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException {
    try {
      return storage
          .objects()
          .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
              target.getContentType() != null ? target : null)
          .setProjection(DEFAULT_PROJECTION)
          .setIfMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
          .setIfMetagenerationNotMatch(IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
          .setIfGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
          .execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public byte[] load(StorageObject from, Map options)
      throws StorageServiceException {
    try {
      Storage.Objects.Get getRequest = storage.objects()
          .get(from.getBucket(), from.getName())
          .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      ByteArrayOutputStream out = new ByteArrayOutputStream();
      getRequest.getMediaHttpDownloader().setDirectDownloadEnabled(true);
      getRequest.executeMediaAndDownloadTo(out);
      return out.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
    com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
    final Map> deletes =
        Maps.newConcurrentMap();
    final Map> updates =
        Maps.newConcurrentMap();
    final Map> gets =
        Maps.newConcurrentMap();
    try {
      for (final Tuple> tuple : request.toDelete) {
        deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toUpdate) {
        patchRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            updates.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      for (final Tuple> tuple : request.toGet) {
        getRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
          @Override
          public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(storageObject, null));
          }

          @Override
          public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
            gets.put(tuple.x(),
                Tuple.of(null, translate(e)));
          }
        });
      }
      batch.execute();
    } catch (IOException ex) {
      throw translate(ex);
    }
    return new BatchResponse(deletes, updates, gets);
  }

  @Override
  public byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException {
    try {
      Get req = storage.objects().get(from.getBucket(), from.getName());
      req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
          .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
          .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
          .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
      MediaHttpDownloader downloader = req.getMediaHttpDownloader();
      // todo: Fix int casting (https://github.com/google/google-api-java-client/issues/937)
      downloader.setContentRange(position, (int) position + bytes);
      downloader.setDirectDownloadEnabled(true);
      ByteArrayOutputStream output = new ByteArrayOutputStream();
      req.executeMediaAndDownloadTo(output);
      return output.toByteArray();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException {
    try {
      GenericUrl url = new GenericUrl(uploadId);
      HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
          new ByteArrayContent(null, toWrite, toWriteOffset, length));
      long limit = destOffset + length;
      StringBuilder range = new StringBuilder("bytes ");
      range.append(destOffset).append('-').append(limit - 1).append('/');
      if (last) {
        range.append(limit);
      } else {
        range.append('*');
      }
      httpRequest.getHeaders().setContentRange(range.toString());
      int code;
      String message;
      IOException exception = null;
      try {
        HttpResponse response = httpRequest.execute();
        code = response.getStatusCode();
        message = response.getStatusMessage();
      } catch (HttpResponseException ex) {
        exception = ex;
        code = ex.getStatusCode();
        message = ex.getStatusMessage();
      }
      if (!last && code != 308 || last && !(code == 200 || code == 201)) {
        if (exception != null) {
          throw exception;
        }
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(code);
        error.setMessage(message);
        throw translate(error);
      }
    } catch (IOException ex) {
      throw translate(ex);
    }
  }

  @Override
  public String open(StorageObject object, Map options)
      throws StorageServiceException {
    try {
      Insert req = storage.objects().insert(object.getBucket(), object);
      GenericUrl url = req.buildHttpRequest().getUrl();
      String scheme = url.getScheme();
      String host = url.getHost();
      String path = "/upload" + url.getRawPath();
      url = new GenericUrl(scheme + "://" + host + path);
      url.set("uploadType", "resumable");
      url.set("name", object.getName());
      for (Option option : options.keySet()) {
        Object content = option.get(options);
        if (content != null) {
          url.set(option.value(), content.toString());
        }
      }
      JsonFactory jsonFactory = storage.getJsonFactory();
      HttpRequestFactory requestFactory = storage.getRequestFactory();
      HttpRequest httpRequest =
          requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
      httpRequest.getHeaders().set("X-Upload-Content-Type",
          MoreObjects.firstNonNull(object.getContentType(), "application/octet-stream"));
      HttpResponse response = httpRequest.execute();
      if (response.getStatusCode() != 200) {
        GoogleJsonError error = new GoogleJsonError();
        error.setCode(response.getStatusCode());
        error.setMessage(response.getStatusMessage());
        throw translate(error);
      }
      return response.getHeaders().getLocation();
    } catch (IOException ex) {
      throw translate(ex);
    }
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java b/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
deleted file mode 100644
index a9882ee231b7..000000000000
--- a/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.common.collect.Iterables;
import com.google.gcloud.datastore.DatastoreServiceOptions;
import com.google.gcloud.storage.StorageServiceOptions;

import java.util.ServiceLoader;

public class ServiceRpcProvider {

  public static DatastoreRpc datastore(DatastoreServiceOptions options) {
    DatastoreRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(DatastoreRpcFactory.class), null);
    return factory == null ? new DefaultDatastoreRpc(options) : factory.create(options);
  }

  public static StorageRpc storage(StorageServiceOptions options) {
    StorageRpcFactory factory =
        Iterables.getFirst(ServiceLoader.load(StorageRpcFactory.class), null);
    return factory == null ? new DefaultStorageRpc(options) : factory.create(options);
  }
}
\ No newline at end of file
diff --git a/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java b/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
deleted file mode 100644
index 75b618f607ae..000000000000
--- a/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.storage.StorageServiceOptions;

public interface StorageRpcFactory extends ServiceRpcFactory {
}
\ No newline at end of file
diff --git a/src/site/apt/index.apt b/src/site/apt/index.apt
index 0d4999bb7764..275ad92934a8 100644
--- a/src/site/apt/index.apt
+++ b/src/site/apt/index.apt
@@ -12,9 +12,7 @@ GCloud Java: Idiomatic Java Client for Google Cloud Platform services.
   
    	* {{{https://github.com/GoogleCloudPlatform/gcloud-java}GitHub repository}}
    	
-   	* {{{http://GoogleCloudPlatform.github.io/gcloud-java/apidocs/index.html}Javadocs}}
-   	
-   	* {{{http://GoogleCloudPlatform.github.io/gcloud-java/dependencies.html}Dependencies}}
+   	* {{./apidocs/index.html}Javadocs}}
    	
    	* {{{https://travis-ci.org/GoogleCloudPlatform/gcloud-java} Continous Integration System (Travis-CI)}}
    	
diff --git a/src/site/site.xml b/src/site/site.xml
index ec2726d99cde..0b23edfd7575 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -24,6 +24,8 @@
   
   
   
-    
+    
+    
+    
   
 

From b2ef42bd446a35de7089809ca55302422e32d65e Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Mon, 25 May 2015 22:23:43 -0700
Subject: [PATCH 235/732] fix index.apt

---
 src/site/apt/index.apt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/site/apt/index.apt b/src/site/apt/index.apt
index 275ad92934a8..395bfe7a65e0 100644
--- a/src/site/apt/index.apt
+++ b/src/site/apt/index.apt
@@ -12,7 +12,7 @@ GCloud Java: Idiomatic Java Client for Google Cloud Platform services.
   
    	* {{{https://github.com/GoogleCloudPlatform/gcloud-java}GitHub repository}}
    	
-   	* {{./apidocs/index.html}Javadocs}}
+   	* {{{./apidocs/index.html}Javadocs}}
    	
    	* {{{https://travis-ci.org/GoogleCloudPlatform/gcloud-java} Continous Integration System (Travis-CI)}}
    	

From 60bcee7bd47cecb45769b9d28ff427bf5bd0f30b Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 12:52:02 -0700
Subject: [PATCH 236/732] update pom descriptions

---
 gcloud-java-core/pom.xml      |  2 +-
 gcloud-java-datastore/pom.xml |  3 +++
 gcloud-java-examples/pom.xml  | 13 -------------
 gcloud-java-storage/pom.xml   |  3 +++
 gcloud-java/pom.xml           | 15 +--------------
 pom.xml                       | 15 ++++++++++-----
 src/site/site.xml             |  2 +-
 7 files changed, 19 insertions(+), 34 deletions(-)

diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml
index 7fbbe7dad21d..78baf824c080 100644
--- a/gcloud-java-core/pom.xml
+++ b/gcloud-java-core/pom.xml
@@ -7,7 +7,7 @@
   GCloud Java core
   https://github.com/GoogleCloudPlatform/gcloud-java
   
-    Java idiomatic client for Google Cloud Platform services.
+    Core module for the gcloud-java.
   
   
     com.google.gcloud
diff --git a/gcloud-java-datastore/pom.xml b/gcloud-java-datastore/pom.xml
index 0217e9cad58c..b890e6d2b755 100644
--- a/gcloud-java-datastore/pom.xml
+++ b/gcloud-java-datastore/pom.xml
@@ -6,6 +6,9 @@
   jar
   GCloud Java datastore
   https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    Java idiomatic client for Google Cloud Datastore.
+  
   
     com.google.gcloud
     gcloud-java-pom
diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml
index ac1ef9f667cb..66d9fc9c93e3 100644
--- a/gcloud-java-examples/pom.xml
+++ b/gcloud-java-examples/pom.xml
@@ -21,17 +21,4 @@
       ${project.version}
     
   
-  
-    
-      
-        org.apache.maven.plugins
-        maven-site-plugin
-        3.1
-        
-          true
-          true
-        
-      
-    
-  
 
diff --git a/gcloud-java-storage/pom.xml b/gcloud-java-storage/pom.xml
index 29731998eeca..11cc0f02c6ac 100644
--- a/gcloud-java-storage/pom.xml
+++ b/gcloud-java-storage/pom.xml
@@ -6,6 +6,9 @@
   jar
   GCloud Java storage
   https://github.com/GoogleCloudPlatform/gcloud-java
+  
+    Java idiomatic client for Google Cloud Storage.
+  
   
     com.google.gcloud
     gcloud-java-pom
diff --git a/gcloud-java/pom.xml b/gcloud-java/pom.xml
index a70763d2b1bd..fb1226a23d5e 100644
--- a/gcloud-java/pom.xml
+++ b/gcloud-java/pom.xml
@@ -4,7 +4,7 @@
   com.google.gcloud
   gcloud-java
   jar
-  GCloud Java assembly
+  GCloud Java
   https://github.com/GoogleCloudPlatform/gcloud-java
   
     Java idiomatic client for Google Cloud Platform services.
@@ -31,17 +31,4 @@
       ${project.version}
     
   
-  
-    
-      
-        org.apache.maven.plugins
-        maven-site-plugin
-        3.1
-        
-          true
-          true
-        
-      
-    
-  
 
diff --git a/pom.xml b/pom.xml
index a00e0d641b30..e6fafbe5e7c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -203,12 +203,18 @@
         org.eluder.coveralls
         coveralls-maven-plugin
         3.0.1
+        
+          
+            ${basedir}/target/coverage.xml
+          
+        
       
       
         org.codehaus.mojo
         cobertura-maven-plugin
         2.6
           
+            ${basedir}/target
             
               xml
               html
@@ -223,7 +229,6 @@
                 com/google/gcloud/examples/**/*.class
               
             
-            
           256m
         
       
@@ -259,15 +264,19 @@
                 
                   
                     index
+                    dependency-info
                     dependencies
+                    dependency-convergence
                     project-team
                     mailing-list
                     cim
                     issue-tracking
                     license
                     scm
+                    dependency-management
                     distribution-management
                     summary
+                    modules
                   
                 
               
@@ -282,10 +291,6 @@
               org.apache.maven.plugins
               maven-dependency-plugin
               2.10
-              
-                true
-                ${project.build.directory}/dependencies
-              
             
             
               org.apache.maven.plugins
diff --git a/src/site/site.xml b/src/site/site.xml
index 0b23edfd7575..55047ce85c54 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -25,7 +25,7 @@
   
   
     
-    
+    
     
   
 

From fdc048bea524591895f94017bd05b17fbcad607a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 14:23:42 -0700
Subject: [PATCH 237/732] remove StorageRpcProvider and add Javadoc to RPC
 factory interfaces

---
 .../com/google/gcloud/ServiceOptions.java     | 11 +++++++
 .../google/gcloud/spi/ServiceRpcFactory.java  |  2 +-
 .../google/gcloud/spi/ServiceRpcProvider.java | 32 -------------------
 .../datastore/DatastoreServiceOptions.java    |  3 +-
 .../gcloud/spi/DatastoreRpcFactory.java       |  4 +++
 .../google/gcloud/spi/StorageRpcFactory.java  |  4 +++
 .../gcloud/storage/StorageServiceOptions.java |  3 +-
 7 files changed, 22 insertions(+), 37 deletions(-)
 delete mode 100644 gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index 448b617372b0..a974a1f1912a 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -25,6 +25,7 @@
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.common.collect.Iterables;
 import com.google.gcloud.spi.ServiceRpcFactory;
 
 import java.io.BufferedReader;
@@ -39,6 +40,7 @@
 import java.net.URL;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -311,4 +313,13 @@ protected boolean isEquals(ServiceOptions other) {
   }
 
   public abstract Builder toBuilder();
+
+  /**
+   * Creates a service RPC using a factory loaded by {@link ServiceLoader}.
+   */
+  protected static > R createRpc(O options,
+      Class> factoryClass) {
+    ServiceRpcFactory factory = Iterables.getFirst(ServiceLoader.load(factoryClass), null);
+    return factory == null ? null : factory.create(options);
+  }
 }
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
index a16d11217bb1..e8e67615305d 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

import java.io.Serializable;

public interface ServiceRpcFactory extends Serializable {

  S create(O options);
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

import java.io.Serializable;

/**
 * A base interface for all service RPC factories.
 * Loading of a factory implementation is done via {@link java.util.ServiceLoader}.
 */
public interface ServiceRpcFactory extends Serializable {

  S create(O options);
}
\ No newline at end of file
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
deleted file mode 100644
index 41a2bf2d5848..000000000000
--- a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcProvider.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.spi;
-
-import com.google.common.collect.Iterables;
-import com.google.gcloud.ServiceOptions;
-
-import java.util.ServiceLoader;
-
-public class ServiceRpcProvider {
-
-  public static > R get(O options,
-      Class> factoryClass) {
-    ServiceRpcFactory factory = Iterables.getFirst(ServiceLoader.load(factoryClass), null);
-    return factory == null ? null : factory.create(options);
-  }
-}
-
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
index 1dc0855baded..e0dba7b7a982 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
@@ -28,7 +28,6 @@
 import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
 import com.google.gcloud.spi.DatastoreRpcFactory;
 import com.google.gcloud.spi.DefaultDatastoreRpc;
-import com.google.gcloud.spi.ServiceRpcProvider;
 
 import java.lang.reflect.Method;
 import java.util.Iterator;
@@ -187,7 +186,7 @@ DatastoreRpc datastoreRpc() {
     if (serviceRpcFactory() != null) {
       datastoreRpc = serviceRpcFactory().create(this);
     } else {
-      datastoreRpc = ServiceRpcProvider.get(this, DatastoreRpcFactory.class);
+      datastoreRpc = createRpc(this, DatastoreRpcFactory.class);
       if (datastoreRpc == null) {
         datastoreRpc = new DefaultDatastoreRpc(this);
       }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
index de9e87e4fa2e..952114788f01 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
@@ -18,6 +18,10 @@
 
 import com.google.gcloud.datastore.DatastoreServiceOptions;
 
+/**
+ * An interface for Datastore RPC factory.
+ * Implementation will be loaded via {@link java.util.ServiceLoader}.
+ */
 public interface DatastoreRpcFactory extends
     ServiceRpcFactory {
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
index d78e721e2331..f693e63018ce 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
@@ -18,6 +18,10 @@
 
 import com.google.gcloud.storage.StorageServiceOptions;
 
+/**
+ * An interface for Storage RPC factory.
+ * Implementation will be loaded via {@link java.util.ServiceLoader}.
+ */
 public interface StorageRpcFactory extends ServiceRpcFactory {
 }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
index a458692511bd..1bb10743de68 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
@@ -20,7 +20,6 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.gcloud.ServiceOptions;
 import com.google.gcloud.spi.DefaultStorageRpc;
-import com.google.gcloud.spi.ServiceRpcProvider;
 import com.google.gcloud.spi.StorageRpc;
 import com.google.gcloud.spi.StorageRpcFactory;
 
@@ -77,7 +76,7 @@ StorageRpc storageRpc() {
     if (serviceRpcFactory() != null) {
       storageRpc = serviceRpcFactory().create(this);
     } else {
-      storageRpc = ServiceRpcProvider.get(this, StorageRpcFactory.class);
+      storageRpc = createRpc(this, StorageRpcFactory.class);
       if (storageRpc == null) {
         storageRpc = new DefaultStorageRpc(this);
       }

From 6012b298e7c4a12804a0492d187cdfd04aaaf3f6 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 14:32:10 -0700
Subject: [PATCH 238/732] fix typo

---
 gcloud-java-core/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcloud-java-core/README.md b/gcloud-java-core/README.md
index e22b159d216f..aa8d4304f1fb 100644
--- a/gcloud-java-core/README.md
+++ b/gcloud-java-core/README.md
@@ -10,7 +10,7 @@ Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
 -  [API Documentation] (http://googlecloudplatform.github.io/gcloud-java/apidocs)
 -  [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/gcloud/examples/package-summary.html)
 
-This moudle provides common functionality and is required by the other service specific modules.
+This module provides common functionality and is required by the other service specific modules.
 
 Quickstart
 ----------

From 8eeae915262b6e7d8426820e525418c9136e3595 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 14:52:42 -0700
Subject: [PATCH 239/732] make chunk size configurable

---
 .../gcloud/examples/StorageExample.java       |  2 +-
 .../gcloud/storage/BlobReadChannel.java       |  7 +++++++
 .../gcloud/storage/BlobReadChannelImpl.java   | 10 ++++++++--
 .../gcloud/storage/BlobWriteChannel.java      |  5 +++++
 .../gcloud/storage/BlobWriterChannelImpl.java | 20 ++++++++++++-------
 .../google/gcloud/storage/StorageService.java |  4 ++--
 6 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
index a4550f19d2de..b0d44c292d2c 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -479,7 +479,7 @@ public String params() {
   }
 
   public static void printUsage() {
-    StringBuilder actionAndParams = new StringBuilder("");
+    StringBuilder actionAndParams = new StringBuilder();
     for (Map.Entry entry : ACTIONS.entrySet()) {
       actionAndParams.append("\n\t").append(entry.getKey());
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
index 89f3420a2a28..ad1a385d9a83 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
@@ -39,4 +39,11 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos
   void close();
 
   void seek(int position) throws IOException;
+
+  /**
+   * Sets the minimum size that will be read by a single RPC.
+   * Read data will be locally buffered until consumed.
+   */
+  void chunkSize(int chunkSize);
+
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
index 8ca8a01f2df3..27d37b127d55 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
@@ -33,7 +33,7 @@
  */
 class BlobReadChannelImpl implements BlobReadChannel {
 
-  private static final int MIN_BUFFER_SIZE = 2 * 1024 * 1024;
+  private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
   private static final long serialVersionUID = 4821762590742862669L;
 
   private final StorageServiceOptions serviceOptions;
@@ -42,6 +42,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private int position;
   private boolean isOpen;
   private boolean endOfStream;
+  private int chunkSize = DEFAULT_CHUNK_SIZE;
 
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
@@ -105,6 +106,11 @@ public void seek(int position) throws IOException {
     endOfStream = false;
   }
 
+  @Override
+  public void chunkSize(int chunkSize) {
+    this.chunkSize = chunkSize <= 0 ? DEFAULT_CHUNK_SIZE : chunkSize;
+  }
+
   @Override
   public int read(ByteBuffer byteBuffer) throws IOException {
     validateOpen();
@@ -112,7 +118,7 @@ public int read(ByteBuffer byteBuffer) throws IOException {
       if (endOfStream) {
         return -1;
       }
-      final int toRead = Math.max(byteBuffer.remaining(), MIN_BUFFER_SIZE);
+      final int toRead = Math.max(byteBuffer.remaining(), chunkSize);
       buffer = runWithRetries(new Callable() {
         @Override
         public byte[] call() {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
index 77ce84a8ea7a..20b2ce087632 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
@@ -29,4 +29,9 @@
  */
 public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable {
 
+  /**
+   * Sets the minimum size that will be written by a single RPC.
+   * Written data will be buffered and only flushed upon reaching this size or closing the channel.
+   */
+  void chunkSize(int chunkSize);
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
index b7736346bba0..2b8e66cc33ce 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
@@ -35,8 +35,8 @@
 class BlobWriterChannelImpl implements BlobWriteChannel {
 
   private static final long serialVersionUID = 8675286882724938737L;
-  private static final int CHUNK_SIZE = 256 * 1024;
-  private static final int MIN_BUFFER_SIZE = 8 * CHUNK_SIZE;
+  private static final int MIN_CHUNK_SIZE = 256 * 1024;
+  private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE;
 
   private final StorageServiceOptions options;
   private final Blob blob;
@@ -45,6 +45,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private byte[] buffer = new byte[0];
   private int limit;
   private boolean isOpen = true;
+  private int chunkSize = DEFAULT_CHUNK_SIZE;
 
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
@@ -65,8 +66,8 @@ private void writeObject(ObjectOutputStream out) throws IOException {
   }
 
   private void flush(boolean compact) {
-    if (limit >= MIN_BUFFER_SIZE || compact && limit >= CHUNK_SIZE) {
-      final int length = limit - limit % CHUNK_SIZE;
+    if (limit >= chunkSize || compact && limit >= MIN_CHUNK_SIZE) {
+      final int length = limit - limit % MIN_CHUNK_SIZE;
       runWithRetries(callable(new Runnable() {
         @Override
         public void run() {
@@ -75,7 +76,7 @@ public void run() {
       }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
       position += length;
       limit -= length;
-      byte[] temp = new byte[compact ? limit : MIN_BUFFER_SIZE];
+      byte[] temp = new byte[compact ? limit : chunkSize];
       System.arraycopy(buffer, length, temp, 0, limit);
       buffer = temp;
     }
@@ -107,8 +108,7 @@ public int write(ByteBuffer byteBuffer) throws IOException {
     if (spaceInBuffer >= toWrite) {
       byteBuffer.get(buffer, limit, toWrite);
     } else {
-      buffer = Arrays.copyOf(buffer,
-          Math.max(MIN_BUFFER_SIZE, buffer.length + toWrite - spaceInBuffer));
+      buffer = Arrays.copyOf(buffer, Math.max(chunkSize, buffer.length + toWrite - spaceInBuffer));
       byteBuffer.get(buffer, limit, toWrite);
     }
     limit += toWrite;
@@ -135,4 +135,10 @@ public void run() {
       buffer = null;
     }
   }
+
+  @Override
+  public void chunkSize(int chunkSize) {
+    chunkSize = (chunkSize / MIN_CHUNK_SIZE) * MIN_CHUNK_SIZE;
+    this.chunkSize = Math.max(MIN_CHUNK_SIZE, chunkSize);
+  }
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
index 9e95b0f1281d..6fefb3af3b16 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -121,8 +121,8 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
       return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
 
-    public static BlobTargetOption doesNotExists() {
-      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0);
+    public static BlobTargetOption doesNotExist() {
+      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L);
     }
 
     public static BlobTargetOption generationMatch() {

From e1edb53f9489b78c869071a0dc7f112fda047946 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 15:18:24 -0700
Subject: [PATCH 240/732] Revert changes that were applied on a different
 branch.

---
 .../com/google/gcloud/AuthCredentials.java    |  12 +-
 .../gcloud/examples/StorageExample.java       |   2 +-
 .../gcloud/storage/BlobReadChannel.java       |   7 +
 .../gcloud/storage/BlobReadChannelImpl.java   |  10 +-
 .../gcloud/storage/BlobWriteChannel.java      |   5 +
 .../gcloud/storage/BlobWriterChannelImpl.java |  20 ++-
 .../java/com/google/gcloud/storage/Cors.java  |  18 +--
 .../com/google/gcloud/storage/HttpMethod.java |  24 +++
 .../google/gcloud/storage/StorageService.java | 152 +++++++++++++++++-
 .../gcloud/storage/StorageServiceImpl.java    |   8 +-
 .../com/google/gcloud/storage/CorsTest.java   |   3 +-
 11 files changed, 233 insertions(+), 28 deletions(-)
 create mode 100644 src/main/java/com/google/gcloud/storage/HttpMethod.java

diff --git a/src/main/java/com/google/gcloud/AuthCredentials.java b/src/main/java/com/google/gcloud/AuthCredentials.java
index 839da54e62cf..6cdb737ddd91 100644
--- a/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -62,7 +62,7 @@ private Object readResolve() throws ObjectStreamException {
     }
   }
 
-  private static class ServiceAccountAuthCredentials extends AuthCredentials {
+  public static class ServiceAccountAuthCredentials extends AuthCredentials {
 
     private static final long serialVersionUID = 8007708734318445901L;
     private final String account;
@@ -94,6 +94,14 @@ protected HttpRequestInitializer httpRequestInitializer(
       return builder.build();
     }
 
+    public String account() {
+      return account;
+    }
+
+    public PrivateKey privateKey() {
+      return privateKey;
+    }
+
     @Override
     public int hashCode() {
       return Objects.hash(account, privateKey);
@@ -187,7 +195,7 @@ public static AuthCredentials createApplicationDefaults() throws IOException {
     return new ApplicationDefaultAuthCredentials();
   }
 
-  public static AuthCredentials createFor(String account, PrivateKey privateKey) {
+  public static ServiceAccountAuthCredentials createFor(String account, PrivateKey privateKey) {
     return new ServiceAccountAuthCredentials(account, privateKey);
   }
 
diff --git a/src/main/java/com/google/gcloud/examples/StorageExample.java b/src/main/java/com/google/gcloud/examples/StorageExample.java
index a4550f19d2de..b0d44c292d2c 100644
--- a/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -479,7 +479,7 @@ public String params() {
   }
 
   public static void printUsage() {
-    StringBuilder actionAndParams = new StringBuilder("");
+    StringBuilder actionAndParams = new StringBuilder();
     for (Map.Entry entry : ACTIONS.entrySet()) {
       actionAndParams.append("\n\t").append(entry.getKey());
 
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
index 89f3420a2a28..ad1a385d9a83 100644
--- a/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannel.java
@@ -39,4 +39,11 @@ public interface BlobReadChannel extends ReadableByteChannel, Serializable, Clos
   void close();
 
   void seek(int position) throws IOException;
+
+  /**
+   * Sets the minimum size that will be read by a single RPC.
+   * Read data will be locally buffered until consumed.
+   */
+  void chunkSize(int chunkSize);
+
 }
diff --git a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
index 8ca8a01f2df3..27d37b127d55 100644
--- a/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
+++ b/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
@@ -33,7 +33,7 @@
  */
 class BlobReadChannelImpl implements BlobReadChannel {
 
-  private static final int MIN_BUFFER_SIZE = 2 * 1024 * 1024;
+  private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
   private static final long serialVersionUID = 4821762590742862669L;
 
   private final StorageServiceOptions serviceOptions;
@@ -42,6 +42,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private int position;
   private boolean isOpen;
   private boolean endOfStream;
+  private int chunkSize = DEFAULT_CHUNK_SIZE;
 
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
@@ -105,6 +106,11 @@ public void seek(int position) throws IOException {
     endOfStream = false;
   }
 
+  @Override
+  public void chunkSize(int chunkSize) {
+    this.chunkSize = chunkSize <= 0 ? DEFAULT_CHUNK_SIZE : chunkSize;
+  }
+
   @Override
   public int read(ByteBuffer byteBuffer) throws IOException {
     validateOpen();
@@ -112,7 +118,7 @@ public int read(ByteBuffer byteBuffer) throws IOException {
       if (endOfStream) {
         return -1;
       }
-      final int toRead = Math.max(byteBuffer.remaining(), MIN_BUFFER_SIZE);
+      final int toRead = Math.max(byteBuffer.remaining(), chunkSize);
       buffer = runWithRetries(new Callable() {
         @Override
         public byte[] call() {
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
index 77ce84a8ea7a..20b2ce087632 100644
--- a/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
+++ b/src/main/java/com/google/gcloud/storage/BlobWriteChannel.java
@@ -29,4 +29,9 @@
  */
 public interface BlobWriteChannel extends WritableByteChannel, Serializable, Closeable {
 
+  /**
+   * Sets the minimum size that will be written by a single RPC.
+   * Written data will be buffered and only flushed upon reaching this size or closing the channel.
+   */
+  void chunkSize(int chunkSize);
 }
diff --git a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
index b7736346bba0..2b8e66cc33ce 100644
--- a/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
+++ b/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
@@ -35,8 +35,8 @@
 class BlobWriterChannelImpl implements BlobWriteChannel {
 
   private static final long serialVersionUID = 8675286882724938737L;
-  private static final int CHUNK_SIZE = 256 * 1024;
-  private static final int MIN_BUFFER_SIZE = 8 * CHUNK_SIZE;
+  private static final int MIN_CHUNK_SIZE = 256 * 1024;
+  private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE;
 
   private final StorageServiceOptions options;
   private final Blob blob;
@@ -45,6 +45,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private byte[] buffer = new byte[0];
   private int limit;
   private boolean isOpen = true;
+  private int chunkSize = DEFAULT_CHUNK_SIZE;
 
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
@@ -65,8 +66,8 @@ private void writeObject(ObjectOutputStream out) throws IOException {
   }
 
   private void flush(boolean compact) {
-    if (limit >= MIN_BUFFER_SIZE || compact && limit >= CHUNK_SIZE) {
-      final int length = limit - limit % CHUNK_SIZE;
+    if (limit >= chunkSize || compact && limit >= MIN_CHUNK_SIZE) {
+      final int length = limit - limit % MIN_CHUNK_SIZE;
       runWithRetries(callable(new Runnable() {
         @Override
         public void run() {
@@ -75,7 +76,7 @@ public void run() {
       }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
       position += length;
       limit -= length;
-      byte[] temp = new byte[compact ? limit : MIN_BUFFER_SIZE];
+      byte[] temp = new byte[compact ? limit : chunkSize];
       System.arraycopy(buffer, length, temp, 0, limit);
       buffer = temp;
     }
@@ -107,8 +108,7 @@ public int write(ByteBuffer byteBuffer) throws IOException {
     if (spaceInBuffer >= toWrite) {
       byteBuffer.get(buffer, limit, toWrite);
     } else {
-      buffer = Arrays.copyOf(buffer,
-          Math.max(MIN_BUFFER_SIZE, buffer.length + toWrite - spaceInBuffer));
+      buffer = Arrays.copyOf(buffer, Math.max(chunkSize, buffer.length + toWrite - spaceInBuffer));
       byteBuffer.get(buffer, limit, toWrite);
     }
     limit += toWrite;
@@ -135,4 +135,10 @@ public void run() {
       buffer = null;
     }
   }
+
+  @Override
+  public void chunkSize(int chunkSize) {
+    chunkSize = (chunkSize / MIN_CHUNK_SIZE) * MIN_CHUNK_SIZE;
+    this.chunkSize = Math.max(MIN_CHUNK_SIZE, chunkSize);
+  }
 }
diff --git a/src/main/java/com/google/gcloud/storage/Cors.java b/src/main/java/com/google/gcloud/storage/Cors.java
index b1953aa5e0e4..ce8cfb95b6e9 100644
--- a/src/main/java/com/google/gcloud/storage/Cors.java
+++ b/src/main/java/com/google/gcloud/storage/Cors.java
@@ -53,14 +53,10 @@ public Bucket.Cors apply(Cors cors) {
   };
 
   private final Integer maxAgeSeconds;
-  private final ImmutableList methods;
+  private final ImmutableList methods;
   private final ImmutableList origins;
   private final ImmutableList responseHeaders;
 
-  public enum Method {
-    ANY, GET, HEAD, PUT, POST, DELETE
-  }
-
   public static final class Origin implements Serializable {
 
     private static final long serialVersionUID = -4447958124895577993L;
@@ -118,7 +114,7 @@ public String value() {
   public static final class Builder {
 
     private Integer maxAgeSeconds;
-    private ImmutableList methods;
+    private ImmutableList methods;
     private ImmutableList origins;
     private ImmutableList responseHeaders;
 
@@ -129,7 +125,7 @@ public Builder maxAgeSeconds(Integer maxAgeSeconds) {
       return this;
     }
 
-    public Builder methods(Iterable methods) {
+    public Builder methods(Iterable methods) {
       this.methods = methods != null ? ImmutableList.copyOf(methods) : null;
       return this;
     }
@@ -160,7 +156,7 @@ public Integer maxAgeSeconds() {
     return maxAgeSeconds;
   }
 
-  public List methods() {
+  public List methods() {
     return methods;
   }
 
@@ -217,10 +213,10 @@ Bucket.Cors toPb() {
   static Cors fromPb(Bucket.Cors cors) {
     Builder builder = builder().maxAgeSeconds(cors.getMaxAgeSeconds());
     if (cors.getMethod() != null) {
-      builder.methods(transform(cors.getMethod(), new Function() {
+      builder.methods(transform(cors.getMethod(), new Function() {
         @Override
-        public Method apply(String name) {
-          return Method.valueOf(name.toUpperCase());
+        public HttpMethod apply(String name) {
+          return HttpMethod.valueOf(name.toUpperCase());
         }
       }));
     }
diff --git a/src/main/java/com/google/gcloud/storage/HttpMethod.java b/src/main/java/com/google/gcloud/storage/HttpMethod.java
new file mode 100644
index 000000000000..f5889aedae90
--- /dev/null
+++ b/src/main/java/com/google/gcloud/storage/HttpMethod.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+/**
+ *
+ */
+public enum HttpMethod {
+  GET, HEAD, PUT, POST, DELETE
+}
diff --git a/src/main/java/com/google/gcloud/storage/StorageService.java b/src/main/java/com/google/gcloud/storage/StorageService.java
index 9e95b0f1281d..2e500b001a93 100644
--- a/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -16,14 +16,19 @@
 
 package com.google.gcloud.storage;
 
+import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.collect.ImmutableList;
+import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.Service;
 import com.google.gcloud.spi.StorageRpc;
 
+import org.joda.time.DateTime;
+
 import java.io.Serializable;
+import java.net.URL;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashSet;
@@ -121,8 +126,8 @@ public static BlobTargetOption predefinedAcl(PredefinedAcl acl) {
       return new BlobTargetOption(StorageRpc.Option.PREDEFINED_ACL, acl.entry());
     }
 
-    public static BlobTargetOption doesNotExists() {
-      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0);
+    public static BlobTargetOption doesNotExist() {
+      return new BlobTargetOption(StorageRpc.Option.IF_GENERATION_MATCH, 0L);
     }
 
     public static BlobTargetOption generationMatch() {
@@ -407,6 +412,139 @@ public static Builder builder() {
     }
   }
 
+  /**
+   * A request for signing a URL.
+   */
+  class SignUrlRequest implements Serializable {
+
+    private final Blob blob;
+    private final HttpMethod httpMethod;
+    private final Long expiration;
+    private final boolean includeContentType;
+    private final boolean includeMd5;
+    private final String headers;
+    private final ServiceAccountAuthCredentials authCredentials;
+
+    public static class Builder {
+
+      private Blob blob;
+      private HttpMethod httpMethod;
+      private long expiration;
+      private boolean includeContentType;
+      private boolean includeMd5;
+      private String headers;
+      private ServiceAccountAuthCredentials authCredentials;
+
+      private Builder() {}
+
+      public Builder blob(Blob blob) {
+        this.blob = blob;
+        return this;
+      }
+
+      /**
+       * The HTTP method to be used with the signed URL.
+       */
+      public Builder httpMethod(HttpMethod httpMethod) {
+        this.httpMethod = httpMethod;
+        return this;
+      }
+
+      /**
+       * Sets expiration time for the URL.
+       * Defaults to one day.
+       */
+      public Builder expiration(long expiration) {
+        this.expiration = expiration;
+        return this;
+      }
+
+      /**
+       * Indicate if signature should include the blob's content-type.
+       * If {@code true} users of the signed URL should include
+       * the same content-type with their request.
+       */
+      public Builder includeContentType(boolean includeContentType) {
+        this.includeContentType = includeContentType;
+        return this;
+      }
+
+      /**
+       * Indicate if signature should include the blob's md5.
+       * If {@code true} users of the signed URL should include it with their requests.
+       */
+      public Builder includeMd5(boolean includeMd5) {
+        this.includeMd5 = includeMd5;
+        return this;
+      }
+
+      /**
+       *  If headers are provided, the server will check to make sure that the client
+       *  provides matching values.
+       *  For information about how to create canonical headers for signing,
+       *  see About Canonical Extension Headers.
+       */
+      public Builder canonicalizedExtensionHeaders(String headers) {
+        this.headers = headers;
+        return this;
+      }
+
+      /**
+       * The service account credentials for signing the URL.
+       */
+      public Builder serviceAccountAuthCredentials(ServiceAccountAuthCredentials authCredentials) {
+        this.authCredentials = authCredentials;
+        return this;
+      }
+
+      public SignUrlRequest build() {
+        return new SignUrlRequest(this);
+      }
+    }
+
+    private SignUrlRequest(Builder builder) {
+      blob = checkNotNull(builder.blob);
+      httpMethod = builder.httpMethod;
+      expiration = firstNonNull(builder.expiration, new DateTime().plusDays(1).getMillis());
+      includeContentType = builder.includeContentType; // verify blob has content-type
+      includeMd5 = builder.includeMd5; // verify blob  has md5
+      authCredentials = builder.authCredentials; // default if null
+      headers = builder.headers;
+    }
+
+    public Blob blob() {
+      return blob;
+    }
+
+    public HttpMethod httpMethod() {
+      return httpMethod;
+    }
+
+    public String canonicalizedExtensionHeaders() {
+      return headers;
+    }
+
+    public long expiration() {
+      return expiration;
+    }
+
+    public ServiceAccountAuthCredentials authCredentials() {
+      return authCredentials;
+    }
+
+    public boolean includeContentType() {
+      return includeContentType;
+    }
+
+    public boolean includeMd5() {
+      return includeMd5;
+    }
+
+    public static Builder builder() {
+      return new Builder();
+    }
+  }
+
   /**
    * Create a new bucket.
    *
@@ -528,4 +666,14 @@ public static Builder builder() {
    * @throws StorageServiceException upon failure
    */
   BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
+
+  /**
+   * Generates a signed URL for a blob.
+   * If you have a blob that you want to allow access to for a set
+   * amount of time, you can use this method to generate a URL that
+   * is only valid within a certain time period.
+   * This is particularly useful if you don't want publicly
+   * accessible blobs, but don't want to require users to explicitly log in.
+   */
+  URL signUrl(SignUrlRequest request);
 }
diff --git a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 95b95141be14..8dda8004c016 100644
--- a/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -29,7 +29,6 @@
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-import static java.util.concurrent.Executors.callable;
 
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Function;
@@ -47,6 +46,7 @@
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
 import java.io.Serializable;
+import java.net.URL;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -429,6 +429,12 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
     return new BlobWriterChannelImpl(options(), blob, optionsMap);
   }
 
+  @Override
+  public URL signUrl(SignUrlRequest request) {
+    // todo: implement and add test
+    return null;
+  }
+
   private Map optionMap(Long generation, Long metaGeneration,
       Iterable options) {
     return optionMap(generation, metaGeneration, options, false);
diff --git a/src/test/java/com/google/gcloud/storage/CorsTest.java b/src/test/java/com/google/gcloud/storage/CorsTest.java
index 8b0379f03583..f29020c6380f 100644
--- a/src/test/java/com/google/gcloud/storage/CorsTest.java
+++ b/src/test/java/com/google/gcloud/storage/CorsTest.java
@@ -19,7 +19,6 @@
 import static org.junit.Assert.assertEquals;
 
 import com.google.common.collect.ImmutableList;
-import com.google.gcloud.storage.Cors.Method;
 import com.google.gcloud.storage.Cors.Origin;
 
 import org.junit.Test;
@@ -39,7 +38,7 @@ public void testOrigin() {
   public void corsTest() {
     List origins = ImmutableList.of(Origin.any(), Origin.of("o"));
     List headers = ImmutableList.of("h1", "h2");
-    List methods = ImmutableList.of(Method.ANY);
+    List methods = ImmutableList.of(HttpMethod.ANY);
     Cors cors = Cors.builder()
         .maxAgeSeconds(100)
         .origins(origins)

From aba5df892a70dcc719317d594b864ba4eaeb5565 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 16:05:15 -0700
Subject: [PATCH 241/732] add HttpMethod which was removed by the merge

---
 .../com/google/gcloud/storage/HttpMethod.java | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java
new file mode 100644
index 000000000000..9d7944140915
--- /dev/null
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/HttpMethod.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+/**
+ * Http method supported by Storage service.
+ */
+public enum HttpMethod {
+  GET, HEAD, PUT, POST, DELETE
+}

From f9ea6ffb5baab00766e45ea55618fc84c5c24f59 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 16:33:55 -0700
Subject: [PATCH 242/732] initial work on removing gcd as a resource

---
 .../gcloud/datastore/LocalGcdHelper.java      | 34 +++++++++++++++----
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 28194a1c9ff4..698d4a223fb2 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -23,6 +23,7 @@
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.FileWriter;
@@ -31,7 +32,10 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -52,8 +56,18 @@ public class LocalGcdHelper {
 
   public static final String DEFAULT_PROJECT_ID = "projectid1";
   public static final int PORT = 8080;
-  private static final String GCD = "gcd-head";
-  private static final String GCD_LOC = '/' + GCD + ".zip";
+  private static final String GCD_VERSION = "gcd-v1beta2-rev1-2.1.2b";
+  private static final String GCD_FILENAME = GCD_VERSION + ".zip";
+  private static final URL GCD_URL;
+
+  static {
+    try {
+      GCD_URL = new URL("http://storage.googleapis.com/gcd/tools/" + GCD_FILENAME);
+    } catch (MalformedURLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
 
   private static class ProcessStreamReader extends Thread {
 
@@ -103,11 +117,17 @@ public LocalGcdHelper(String projectId) {
 
   public void start() throws IOException, InterruptedException {
     sendQuitRequest();
-    gcdPath = Files.createTempDirectory("gcd");
+    gcdPath = Files.createTempDirectory(GCD_VERSION);
     File gcdFolder = gcdPath.toFile();
     gcdFolder.deleteOnExit();
 
-    try (ZipInputStream zipIn = new ZipInputStream(getClass().getResourceAsStream(GCD_LOC))) {
+    File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME);
+    if (!gcdZipFile.exists()) {
+      ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream());
+      FileOutputStream fos = new FileOutputStream(gcdZipFile);
+      fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+    }
+    try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) {
       ZipEntry entry = zipIn.getNextEntry();
       while (entry != null) {
         File filePath = new File(gcdFolder, entry.getName());
@@ -121,20 +141,20 @@ public void start() throws IOException, InterruptedException {
       }
     }
 
-    File datasetFolder = new File(gcdFolder, GCD + '/' + projectId);
+    File datasetFolder = new File(gcdFolder, GCD_VERSION + '/' + projectId);
     deleteRecurse(datasetFolder.toPath());
 
     // TODO: if System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
     Process temp = new ProcessBuilder()
         .redirectErrorStream(true)
-        .directory(new File(gcdFolder, GCD))
+        .directory(new File(gcdFolder, GCD_VERSION))
         .redirectOutput(new File("/dev/null"))
         .command("bash", "gcd.sh", "create", "-d", projectId, projectId)
         .start();
     temp.waitFor();
 
     temp = new ProcessBuilder()
-        .directory(new File(gcdFolder, GCD))
+        .directory(new File(gcdFolder, GCD_VERSION))
         .redirectErrorStream(true)
         .command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", projectId)
         .start();

From bdf93a09ec55979d7e0d2ac62c9d4d69cc9674c8 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Tue, 26 May 2015 17:15:59 -0700
Subject: [PATCH 243/732] continue work on replacing local gcd.sh with a remote
 version

---
 .../gcloud/datastore/LocalGcdHelper.java      | 40 +++++++++++++------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 698d4a223fb2..87f034a5ed18 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -42,6 +42,7 @@
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Locale;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -117,7 +118,7 @@ public LocalGcdHelper(String projectId) {
 
   public void start() throws IOException, InterruptedException {
     sendQuitRequest();
-    gcdPath = Files.createTempDirectory(GCD_VERSION);
+    gcdPath = Files.createTempDirectory("gcd");
     File gcdFolder = gcdPath.toFile();
     gcdFolder.deleteOnExit();
 
@@ -126,6 +127,7 @@ public void start() throws IOException, InterruptedException {
       ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream());
       FileOutputStream fos = new FileOutputStream(gcdZipFile);
       fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+      fos.close();
     }
     try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) {
       ZipEntry entry = zipIn.getNextEntry();
@@ -140,27 +142,41 @@ public void start() throws IOException, InterruptedException {
         entry = zipIn.getNextEntry();
       }
     }
-
     File datasetFolder = new File(gcdFolder, GCD_VERSION + '/' + projectId);
     deleteRecurse(datasetFolder.toPath());
 
-    // TODO: if System.getProperty("os.name").startsWith("Windows") use cmd.exe /c and gcd.cmd
-    Process temp = new ProcessBuilder()
+    ProcessBuilder processBuilder = new ProcessBuilder()
         .redirectErrorStream(true)
-        .directory(new File(gcdFolder, GCD_VERSION))
-        .redirectOutput(new File("/dev/null"))
-        .command("bash", "gcd.sh", "create", "-d", projectId, projectId)
-        .start();
+        .directory(new File(gcdFolder, GCD_VERSION));
+    if (isWindows()) {
+      processBuilder.command("cmd", "/C", "gcd.cmd", "create", "-p", projectId, projectId);
+      processBuilder.redirectOutput(new File("NULL:"));
+    } else {
+      processBuilder.redirectOutput(new File("/dev/null"));
+      processBuilder.command("bash", "gcd.sh", "create", "-p", projectId, projectId);
+    }
+
+    Process temp = processBuilder.start();
     temp.waitFor();
 
-    temp = new ProcessBuilder()
+    processBuilder = new ProcessBuilder()
         .directory(new File(gcdFolder, GCD_VERSION))
-        .redirectErrorStream(true)
-        .command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown", projectId)
-        .start();
+        .redirectErrorStream(true);
+    if (isWindows()) {
+      processBuilder.command("cmd", "/C", "gcd.cmd", "start", "--testing",
+          "--allow_remote_shutdown", projectId);
+    } else {
+      processBuilder.command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown",
+          projectId);
+    }
+    temp = processBuilder.start();
     processReader = ProcessStreamReader.start(temp, "Dev App Server is now running");
   }
 
+  private static boolean isWindows() {
+    return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).indexOf("windows") > -1;
+  }
+
   private static void extractFile(ZipInputStream zipIn, File filePath) throws IOException {
     try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) {
       byte[] bytesIn = new byte[1024];

From d84b8b8f538e8d94aa6eee4056b0e827a39447af Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 26 May 2015 19:27:49 -0700
Subject: [PATCH 244/732] add some comments describing how the LocalGcdHelper
 works

---
 .../google/gcloud/datastore/LocalGcdHelper.java   | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 87f034a5ed18..f8e11885a62d 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -116,12 +116,23 @@ public LocalGcdHelper(String projectId) {
     this.projectId = projectId;
   }
 
+  /**
+   * Starts the local datastore for the specific project.
+   *
+   * This will unzip the gcd tool, create the project and start it.
+   * All content is written to a temporary directory that will be deleted when
+   * {@link #stop()} is called or when the program terminates) to make sure that no left-over
+   * data from prior runs is used.
+   */
   public void start() throws IOException, InterruptedException {
+    // send a quick request in case we have a hanging process from a previous run
     sendQuitRequest();
+    // Each run is associated with its own folder that is deleted once test completes.
     gcdPath = Files.createTempDirectory("gcd");
     File gcdFolder = gcdPath.toFile();
     gcdFolder.deleteOnExit();
 
+    // check if we already have a local copy of the gcd utility and download it if not.
     File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME);
     if (!gcdZipFile.exists()) {
       ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream());
@@ -129,6 +140,7 @@ public void start() throws IOException, InterruptedException {
       fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
       fos.close();
     }
+    // unzip the gcd
     try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) {
       ZipEntry entry = zipIn.getNextEntry();
       while (entry != null) {
@@ -142,9 +154,11 @@ public void start() throws IOException, InterruptedException {
         entry = zipIn.getNextEntry();
       }
     }
+    // cleanup any possible data for the same project
     File datasetFolder = new File(gcdFolder, GCD_VERSION + '/' + projectId);
     deleteRecurse(datasetFolder.toPath());
 
+    // create the datastore for the project
     ProcessBuilder processBuilder = new ProcessBuilder()
         .redirectErrorStream(true)
         .directory(new File(gcdFolder, GCD_VERSION));
@@ -159,6 +173,7 @@ public void start() throws IOException, InterruptedException {
     Process temp = processBuilder.start();
     temp.waitFor();
 
+    // start the datastore for the project
     processBuilder = new ProcessBuilder()
         .directory(new File(gcdFolder, GCD_VERSION))
         .redirectErrorStream(true);

From d04ac4c348949f9f32294edd7ff8b319fb390ad9 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 26 May 2015 19:45:40 -0700
Subject: [PATCH 245/732] display stderr of gcd create

---
 .../test/java/com/google/gcloud/datastore/LocalGcdHelper.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index f8e11885a62d..cca7e25697f0 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -160,7 +160,7 @@ public void start() throws IOException, InterruptedException {
 
     // create the datastore for the project
     ProcessBuilder processBuilder = new ProcessBuilder()
-        .redirectErrorStream(true)
+        .redirectError(ProcessBuilder.Redirect.INHERIT)
         .directory(new File(gcdFolder, GCD_VERSION));
     if (isWindows()) {
       processBuilder.command("cmd", "/C", "gcd.cmd", "create", "-p", projectId, projectId);

From 7a924870204bb9ae47f61c006e2b008b828a70fe Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 26 May 2015 20:31:13 -0700
Subject: [PATCH 246/732] work on sign url

---
 .../com/google/gcloud/spi/StorageRpc.java     |   2 +-
 .../google/gcloud/storage/StorageService.java | 190 +++++-------------
 .../gcloud/storage/StorageServiceImpl.java    |   2 +-
 3 files changed, 53 insertions(+), 141 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
index ab1e9affbbce..5a99cce69aa5 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  // These options are part of the Google Cloud storage header options
  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException;
}
\ No newline at end of file
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
index 2e500b001a93..27c7f8636be6 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -16,19 +16,16 @@
 
 package com.google.gcloud.storage;
 
-import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.collect.ImmutableList;
-import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.Service;
 import com.google.gcloud.spi.StorageRpc;
 
-import org.joda.time.DateTime;
-
 import java.io.Serializable;
 import java.net.URL;
+import java.security.PrivateKey;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashSet;
@@ -218,6 +215,51 @@ public static BlobListOption recursive(boolean recursive) {
     }
   }
 
+  class SignUrlOption implements Serializable {
+
+    private static final long serialVersionUID = 7850569877451099267L;
+
+    private final String name;
+    private final Object value;
+
+    private SignUrlOption(String name, Object value) {
+      this.name = name;
+      this.value = value;
+    }
+
+    /**
+     * The HTTP method to be used with the signed URL.
+     */
+    public static SignUrlOption httpMethod(HttpMethod httpMethod) {
+      return new SignUrlOption("HTTP_METHOD", httpMethod.name());
+    }
+
+    /**
+     * Use it if signature should include the blob's content-type.
+     * When used, users of the signed URL should include the blob's content-type with their request.
+     */
+    public static SignUrlOption withContentType() {
+      return new SignUrlOption("CONTENT_TYPE", true);
+    }
+
+    /**
+     * Use it if signature should include the blob's md5.
+     * When used, users of the signed URL should include the blob's md5 with their request.
+     */
+    public static SignUrlOption withMd5() {
+      return new SignUrlOption("MD5", true);
+    }
+
+    /**
+     * The private key to use for signing the URL.
+     * A private key is required for signing. If not provided an attempt will be made to get
+     * it from the service credentials.
+     */
+    public static SignUrlOption signWith(PrivateKey privateKey) {
+      return new SignUrlOption("PRIVATE_KEY", privateKey);
+    }
+  }
+
   class ComposeRequest implements Serializable {
 
     private static final long serialVersionUID = -7385681353748590911L;
@@ -412,139 +454,6 @@ public static Builder builder() {
     }
   }
 
-  /**
-   * A request for signing a URL.
-   */
-  class SignUrlRequest implements Serializable {
-
-    private final Blob blob;
-    private final HttpMethod httpMethod;
-    private final Long expiration;
-    private final boolean includeContentType;
-    private final boolean includeMd5;
-    private final String headers;
-    private final ServiceAccountAuthCredentials authCredentials;
-
-    public static class Builder {
-
-      private Blob blob;
-      private HttpMethod httpMethod;
-      private long expiration;
-      private boolean includeContentType;
-      private boolean includeMd5;
-      private String headers;
-      private ServiceAccountAuthCredentials authCredentials;
-
-      private Builder() {}
-
-      public Builder blob(Blob blob) {
-        this.blob = blob;
-        return this;
-      }
-
-      /**
-       * The HTTP method to be used with the signed URL.
-       */
-      public Builder httpMethod(HttpMethod httpMethod) {
-        this.httpMethod = httpMethod;
-        return this;
-      }
-
-      /**
-       * Sets expiration time for the URL.
-       * Defaults to one day.
-       */
-      public Builder expiration(long expiration) {
-        this.expiration = expiration;
-        return this;
-      }
-
-      /**
-       * Indicate if signature should include the blob's content-type.
-       * If {@code true} users of the signed URL should include
-       * the same content-type with their request.
-       */
-      public Builder includeContentType(boolean includeContentType) {
-        this.includeContentType = includeContentType;
-        return this;
-      }
-
-      /**
-       * Indicate if signature should include the blob's md5.
-       * If {@code true} users of the signed URL should include it with their requests.
-       */
-      public Builder includeMd5(boolean includeMd5) {
-        this.includeMd5 = includeMd5;
-        return this;
-      }
-
-      /**
-       *  If headers are provided, the server will check to make sure that the client
-       *  provides matching values.
-       *  For information about how to create canonical headers for signing,
-       *  see About Canonical Extension Headers.
-       */
-      public Builder canonicalizedExtensionHeaders(String headers) {
-        this.headers = headers;
-        return this;
-      }
-
-      /**
-       * The service account credentials for signing the URL.
-       */
-      public Builder serviceAccountAuthCredentials(ServiceAccountAuthCredentials authCredentials) {
-        this.authCredentials = authCredentials;
-        return this;
-      }
-
-      public SignUrlRequest build() {
-        return new SignUrlRequest(this);
-      }
-    }
-
-    private SignUrlRequest(Builder builder) {
-      blob = checkNotNull(builder.blob);
-      httpMethod = builder.httpMethod;
-      expiration = firstNonNull(builder.expiration, new DateTime().plusDays(1).getMillis());
-      includeContentType = builder.includeContentType; // verify blob has content-type
-      includeMd5 = builder.includeMd5; // verify blob  has md5
-      authCredentials = builder.authCredentials; // default if null
-      headers = builder.headers;
-    }
-
-    public Blob blob() {
-      return blob;
-    }
-
-    public HttpMethod httpMethod() {
-      return httpMethod;
-    }
-
-    public String canonicalizedExtensionHeaders() {
-      return headers;
-    }
-
-    public long expiration() {
-      return expiration;
-    }
-
-    public ServiceAccountAuthCredentials authCredentials() {
-      return authCredentials;
-    }
-
-    public boolean includeContentType() {
-      return includeContentType;
-    }
-
-    public boolean includeMd5() {
-      return includeMd5;
-    }
-
-    public static Builder builder() {
-      return new Builder();
-    }
-  }
-
   /**
    * Create a new bucket.
    *
@@ -669,11 +578,14 @@ public static Builder builder() {
 
   /**
    * Generates a signed URL for a blob.
-   * If you have a blob that you want to allow access to for a set
+   * If you have a blob that you want to allow access to for a fixed
    * amount of time, you can use this method to generate a URL that
    * is only valid within a certain time period.
    * This is particularly useful if you don't want publicly
    * accessible blobs, but don't want to require users to explicitly log in.
+   *
+   * @param blob the blob associated with the signed url
+   * @param  expiration the signed URL expiration (epoch time in milliseconds)
    */
-  URL signUrl(SignUrlRequest request);
+  URL signUrl(Blob blob, long expiration, SignUrlOption... options);
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 8dda8004c016..17d603372a01 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -430,7 +430,7 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
   }
 
   @Override
-  public URL signUrl(SignUrlRequest request) {
+  public URL signUrl(Blob blob, long expiration, SignUrlOption... options) {
     // todo: implement and add test
     return null;
   }

From 716c2eb7df6bdcde8e50139b45a9875168cf6680 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 27 May 2015 09:48:45 -0700
Subject: [PATCH 247/732] replace GCD_VERSION with GCD

---
 .../com/google/gcloud/datastore/LocalGcdHelper.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index cca7e25697f0..47cd63b19ab0 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -57,8 +57,8 @@ public class LocalGcdHelper {
 
   public static final String DEFAULT_PROJECT_ID = "projectid1";
   public static final int PORT = 8080;
-  private static final String GCD_VERSION = "gcd-v1beta2-rev1-2.1.2b";
-  private static final String GCD_FILENAME = GCD_VERSION + ".zip";
+  private static final String GCD = "gcd-v1beta2-rev1-2.1.2b";
+  private static final String GCD_FILENAME = GCD + ".zip";
   private static final URL GCD_URL;
 
   static {
@@ -155,13 +155,13 @@ public void start() throws IOException, InterruptedException {
       }
     }
     // cleanup any possible data for the same project
-    File datasetFolder = new File(gcdFolder, GCD_VERSION + '/' + projectId);
+    File datasetFolder = new File(gcdFolder, GCD + '/' + projectId);
     deleteRecurse(datasetFolder.toPath());
 
     // create the datastore for the project
     ProcessBuilder processBuilder = new ProcessBuilder()
         .redirectError(ProcessBuilder.Redirect.INHERIT)
-        .directory(new File(gcdFolder, GCD_VERSION));
+        .directory(new File(gcdFolder, GCD));
     if (isWindows()) {
       processBuilder.command("cmd", "/C", "gcd.cmd", "create", "-p", projectId, projectId);
       processBuilder.redirectOutput(new File("NULL:"));
@@ -175,7 +175,7 @@ public void start() throws IOException, InterruptedException {
 
     // start the datastore for the project
     processBuilder = new ProcessBuilder()
-        .directory(new File(gcdFolder, GCD_VERSION))
+        .directory(new File(gcdFolder, GCD))
         .redirectErrorStream(true);
     if (isWindows()) {
       processBuilder.command("cmd", "/C", "gcd.cmd", "start", "--testing",

From eefd171cb3573b493f259a84ee23b1351af21229 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 27 May 2015 10:50:31 -0700
Subject: [PATCH 248/732] add a checksum verification

---
 .../gcloud/datastore/LocalGcdHelper.java      | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 47cd63b19ab0..7ed9a4e830cf 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -20,6 +20,7 @@
 
 import com.google.common.base.Strings;
 
+import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.File;
@@ -31,6 +32,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.math.BigInteger;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -42,6 +44,8 @@
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Locale;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -59,6 +63,7 @@ public class LocalGcdHelper {
   public static final int PORT = 8080;
   private static final String GCD = "gcd-v1beta2-rev1-2.1.2b";
   private static final String GCD_FILENAME = GCD + ".zip";
+  private static final String MD5_CHECKSUM = "d84384cdfa8658e1204f4f8be51300e8";
   private static final URL GCD_URL;
 
   static {
@@ -69,7 +74,6 @@ public class LocalGcdHelper {
     }
   }
 
-
   private static class ProcessStreamReader extends Thread {
 
     private final Process process;
@@ -134,7 +138,7 @@ public void start() throws IOException, InterruptedException {
 
     // check if we already have a local copy of the gcd utility and download it if not.
     File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME);
-    if (!gcdZipFile.exists()) {
+    if (!gcdZipFile.exists() || !MD5_CHECKSUM.equals(md5(gcdZipFile))) {
       ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream());
       FileOutputStream fos = new FileOutputStream(gcdZipFile);
       fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
@@ -188,6 +192,22 @@ public void start() throws IOException, InterruptedException {
     processReader = ProcessStreamReader.start(temp, "Dev App Server is now running");
   }
 
+  private static String md5(File gcdZipFile) throws IOException {
+    try {
+      MessageDigest md5 = MessageDigest.getInstance("MD5");
+      try (InputStream is = new BufferedInputStream(new FileInputStream(gcdZipFile))) {
+        byte[] bytes = new byte[4 * 1024 * 1024];
+        int len;
+        while ((len = is.read(bytes)) >= 0) {
+          md5.update(bytes, 0, len);
+        }
+      }
+      return String.format("%032x",new BigInteger(1, md5.digest()));
+    } catch (NoSuchAlgorithmException e) {
+      throw new IOException(e);
+    }
+  }
+
   private static boolean isWindows() {
     return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).indexOf("windows") > -1;
   }

From 86d311fe523f6f0cb092f8386aa39bbac2ded4df Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Wed, 27 May 2015 18:00:40 -0700
Subject: [PATCH 249/732] sign url - work in progress

---
 .../google/gcloud/storage/StorageService.java    | 16 ++++++++++++++--
 .../gcloud/storage/StorageServiceImpl.java       |  7 ++++++-
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
index 27c7f8636be6..5f65b66b8c20 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -222,11 +222,23 @@ class SignUrlOption implements Serializable {
     private final String name;
     private final Object value;
 
+    enum OPTIONS {
+
+    }
+    
     private SignUrlOption(String name, Object value) {
       this.name = name;
       this.value = value;
     }
 
+    String name() {
+      return name;
+    }
+
+    Object value() {
+      return value;
+    }
+
     /**
      * The HTTP method to be used with the signed URL.
      */
@@ -585,7 +597,7 @@ public static Builder builder() {
    * accessible blobs, but don't want to require users to explicitly log in.
    *
    * @param blob the blob associated with the signed url
-   * @param  expiration the signed URL expiration (epoch time in milliseconds)
+   * @param  expirationTimeMillis the signed URL expiration (epoch time in milliseconds)
    */
-  URL signUrl(Blob blob, long expiration, SignUrlOption... options);
+  URL signUrl(Blob blob, long expirationTimeMillis, SignUrlOption... options);
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 17d603372a01..508092e78db5 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -47,6 +47,7 @@
 
 import java.io.Serializable;
 import java.net.URL;
+import java.security.PrivateKey;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -431,7 +432,11 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
 
   @Override
   public URL signUrl(Blob blob, long expiration, SignUrlOption... options) {
-    // todo: implement and add test
+    Map optionMap = Maps.newHashMapWithExpectedSize(options.length);
+    for (SignUrlOption option : options) {
+      optionMap.put(option.name(), option.value());
+    }
+    PrivateKey key = (PrivateKey) optionMap.get("");
     return null;
   }
 

From ebb069c74e728955fa90f7ac2e064f57cbdc625a Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Wed, 27 May 2015 22:11:32 -0700
Subject: [PATCH 250/732] sign url - work in progress

---
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../google/gcloud/storage/StorageService.java | 42 ++++++-----
 .../gcloud/storage/StorageServiceImpl.java    | 73 +++++++++++++++++--
 3 files changed, 89 insertions(+), 28 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index e27d837d7173..12c74b811ea9 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -81,10 +81,10 @@ public DefaultStorageRpc(StorageServiceOptions options) {
     HttpTransport transport = options.httpTransportFactory().create();
     HttpRequestInitializer initializer = options.httpRequestInitializer();
     this.options = options;
+    // todo: validate what is returned by getRootURL and use that for host default (and use host())
     storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
         .setApplicationName("gcloud-java")
         .build();
-    // Todo: make sure nulls are being used as Data.asNull()
   }
 
   private static StorageServiceException translate(IOException exception) {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
index 5f65b66b8c20..edf12c9f8feb 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
@@ -20,12 +20,12 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.common.collect.ImmutableList;
+import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.Service;
 import com.google.gcloud.spi.StorageRpc;
 
 import java.io.Serializable;
 import java.net.URL;
-import java.security.PrivateKey;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashSet;
@@ -219,20 +219,20 @@ class SignUrlOption implements Serializable {
 
     private static final long serialVersionUID = 7850569877451099267L;
 
-    private final String name;
+    private final Option option;
     private final Object value;
 
-    enum OPTIONS {
-
+    enum Option {
+      HTTP_METHOD, CONTENT_TYPE, MD5, SERVICE_ACCOUNT_CRED;
     }
-    
-    private SignUrlOption(String name, Object value) {
-      this.name = name;
+
+    private SignUrlOption(Option option, Object value) {
+      this.option = option;
       this.value = value;
     }
 
-    String name() {
-      return name;
+    Option option() {
+      return option;
     }
 
     Object value() {
@@ -243,7 +243,7 @@ Object value() {
      * The HTTP method to be used with the signed URL.
      */
     public static SignUrlOption httpMethod(HttpMethod httpMethod) {
-      return new SignUrlOption("HTTP_METHOD", httpMethod.name());
+      return new SignUrlOption(Option.HTTP_METHOD, httpMethod.name());
     }
 
     /**
@@ -251,7 +251,7 @@ public static SignUrlOption httpMethod(HttpMethod httpMethod) {
      * When used, users of the signed URL should include the blob's content-type with their request.
      */
     public static SignUrlOption withContentType() {
-      return new SignUrlOption("CONTENT_TYPE", true);
+      return new SignUrlOption(Option.CONTENT_TYPE, true);
     }
 
     /**
@@ -259,16 +259,17 @@ public static SignUrlOption withContentType() {
      * When used, users of the signed URL should include the blob's md5 with their request.
      */
     public static SignUrlOption withMd5() {
-      return new SignUrlOption("MD5", true);
+      return new SignUrlOption(Option.MD5, true);
     }
 
     /**
-     * The private key to use for signing the URL.
-     * A private key is required for signing. If not provided an attempt will be made to get
-     * it from the service credentials.
+     * Service account credentials which are used for signing the URL.
+     * If not provided an attempt will be made to get it from the environment.
+     *
+     * @see Service account
      */
-    public static SignUrlOption signWith(PrivateKey privateKey) {
-      return new SignUrlOption("PRIVATE_KEY", privateKey);
+    public static SignUrlOption serviceAccount(ServiceAccountAuthCredentials credentials) {
+      return new SignUrlOption(Option.SERVICE_ACCOUNT_CRED, credentials);
     }
   }
 
@@ -278,7 +279,7 @@ class ComposeRequest implements Serializable {
 
     private final List sourceBlobs;
     private final Blob target;
-    private final List targetOptions;
+    private final List targetOptions;
 
     public static class SourceBlob implements Serializable {
 
@@ -597,7 +598,8 @@ public static Builder builder() {
    * accessible blobs, but don't want to require users to explicitly log in.
    *
    * @param blob the blob associated with the signed url
-   * @param  expirationTimeMillis the signed URL expiration (epoch time in milliseconds)
+   * @param  expirationTimeInSeconds the signed URL expiration (using epoch time)
+   * @see Signed-URLs
    */
-  URL signUrl(Blob blob, long expirationTimeMillis, SignUrlOption... options);
+  URL signUrl(Blob blob, long expirationTimeInSeconds, SignUrlOption... options);
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 508092e78db5..340b473cb0f3 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -38,7 +38,9 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.io.BaseEncoding;
 import com.google.common.primitives.Ints;
+import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
 import com.google.gcloud.ExceptionHandler.Interceptor;
@@ -46,9 +48,15 @@
 import com.google.gcloud.spi.StorageRpc.Tuple;
 
 import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
 import java.net.URL;
-import java.security.PrivateKey;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.Signature;
+import java.security.SignatureException;
 import java.util.Arrays;
+import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -83,10 +91,8 @@ public RetryResult beforeEval(Exception exception) {
   StorageServiceImpl(StorageServiceOptions options) {
     super(options);
     storageRpc = options.storageRpc();
-    // todo: replace nulls with Value.asNull (per toPb)
     // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors
     // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite
-    // todo: provide signed urls - https://cloud.google.com/storage/docs/access-control#Signed-URLs
     // todo: check if we need to expose https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert vs using bucket update/patch
   }
 
@@ -432,12 +438,65 @@ public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
 
   @Override
   public URL signUrl(Blob blob, long expiration, SignUrlOption... options) {
-    Map optionMap = Maps.newHashMapWithExpectedSize(options.length);
+    EnumMap optionMap = Maps.newEnumMap(SignUrlOption.Option.class);
     for (SignUrlOption option : options) {
-      optionMap.put(option.name(), option.value());
+      optionMap.put(option.option(), option.value());
+    }
+    ServiceAccountAuthCredentials cred =
+        (ServiceAccountAuthCredentials) optionMap.get(SignUrlOption.Option.SERVICE_ACCOUNT_CRED);
+    if (cred == null) {
+      checkArgument(options().authCredentials() instanceof ServiceAccountAuthCredentials,
+          "Signing key was not provided and could not be derived");
+      cred = (ServiceAccountAuthCredentials) this.options().authCredentials();
+    }
+    // construct signature data - see https://cloud.google.com/storage/docs/access-control#Signed-URLs
+    StringBuilder stBuilder = new StringBuilder();
+    if (optionMap.containsKey(SignUrlOption.Option.HTTP_METHOD)) {
+      stBuilder.append(optionMap.get(SignUrlOption.Option.HTTP_METHOD));
+    } else {
+      stBuilder.append(HttpMethod.GET);
+    }
+    stBuilder.append('\n');
+    if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5) , false)) {
+      checkArgument(blob.md5() != null, "Blob is missing a value for md5");
+      stBuilder.append(blob.md5());
+    }
+    stBuilder.append('\n');
+    if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE) , false)) {
+      checkArgument(blob.contentType() != null, "Blob is missing a value for content-type");
+      stBuilder.append(blob.contentType());
+    }
+    stBuilder.append('\n');
+    stBuilder.append(expiration).append('\n').append('\n');
+    StringBuilder path = new StringBuilder();
+    if (!blob.bucket().startsWith("/")) {
+      path.append('/');
+    }
+    path.append(blob.bucket());
+    if (!blob.bucket().endsWith("/")) {
+      path.append('/');
+    }
+    if (blob.name().startsWith("/")) {
+      path.setLength(stBuilder.length() - 1);
+    }
+    path.append(blob.name());
+    stBuilder.append(path);
+    try {
+      Signature signer = Signature.getInstance("SHA256withRSA");
+      signer.initSign(cred.privateKey());
+      signer.update(stBuilder.toString().getBytes("UTF-8"));
+      String signature = BaseEncoding.base64Url().encode(signer.sign());
+      // todo - use options().host() - after default is correct and value is past to RPC
+      stBuilder = new StringBuilder("https://storage.googleapis.com").append(path);
+      stBuilder.append("?GoogleAccessId=").append(cred.account());
+      stBuilder.append("&Expires=").append(expiration);
+      stBuilder.append("&Signature=").append(signature);
+      return new URL(stBuilder.toString());
+    } catch (MalformedURLException | NoSuchAlgorithmException | UnsupportedEncodingException e) {
+      throw new IllegalStateException(e);
+    } catch (SignatureException | InvalidKeyException e) {
+      throw new IllegalArgumentException("Invalid service account private key");
     }
-    PrivateKey key = (PrivateKey) optionMap.get("");
-    return null;
   }
 
   private Map optionMap(Long generation, Long metaGeneration,

From 99559988253581e283a9eda893cf91501d25504a Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Thu, 28 May 2015 15:50:02 -0700
Subject: [PATCH 251/732] complete work on signURL

---
 gcloud-java-core/pom.xml                      |  4 +-
 gcloud-java-examples/pom.xml                  | 11 ++++
 .../gcloud/examples/StorageExample.java       | 66 +++++++++++++++++--
 .../google/gcloud/spi/DefaultStorageRpc.java  |  2 +-
 .../gcloud/storage/StorageServiceImpl.java    | 10 +--
 .../com/google/gcloud/storage/CorsTest.java   |  2 +-
 pom.xml                                       | 13 ++++
 7 files changed, 96 insertions(+), 12 deletions(-)

diff --git a/gcloud-java-core/pom.xml b/gcloud-java-core/pom.xml
index 78baf824c080..fa2e1c18972f 100644
--- a/gcloud-java-core/pom.xml
+++ b/gcloud-java-core/pom.xml
@@ -28,13 +28,13 @@
     
       com.google.http-client
       google-http-client
-      1.19.0
+      1.20.0
       compile
     
     
       com.google.oauth-client
       google-oauth-client
-      1.19.0
+      1.20.0
       compile
     
     
diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml
index 66d9fc9c93e3..1c0357d63635 100644
--- a/gcloud-java-examples/pom.xml
+++ b/gcloud-java-examples/pom.xml
@@ -21,4 +21,15 @@
       ${project.version}
     
   
+  
+    
+      
+        org.codehaus.mojo
+        exec-maven-plugin
+        
+          false
+        
+      
+    
+  
 
diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
index b0d44c292d2c..ccf2cd6f5b76 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -16,6 +16,8 @@
 
 package com.google.gcloud.examples;
 
+import com.google.gcloud.AuthCredentials;
+import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials;
 import com.google.gcloud.RetryParams;
 import com.google.gcloud.spi.StorageRpc.Tuple;
 import com.google.gcloud.storage.BatchRequest;
@@ -27,6 +29,7 @@
 import com.google.gcloud.storage.StorageService;
 import com.google.gcloud.storage.StorageService.ComposeRequest;
 import com.google.gcloud.storage.StorageService.CopyRequest;
+import com.google.gcloud.storage.StorageService.SignUrlOption;
 import com.google.gcloud.storage.StorageServiceFactory;
 import com.google.gcloud.storage.StorageServiceOptions;
 
@@ -40,7 +43,14 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -58,7 +68,8 @@
  * -Dexec.args="[] list []| info [ []]|
  *  download   [local_file]| upload   []|
  *  delete  +| cp    |
- *  compose  + | update_metadata   [key=value]*"}
+ *  compose  + | update_metadata   [key=value]*|
+ *  sign_url    "}
  * 
  * 
  *
@@ -75,7 +86,7 @@ private static abstract class StorageAction {
 
     abstract void run(StorageService storage, T request) throws Exception;
 
-    abstract T parse(String... args) throws IllegalArgumentException, IOException;
+    abstract T parse(String... args) throws Exception;
 
     protected String params() {
       return "";
@@ -424,7 +435,7 @@ public String params() {
    *
    * @see Objects: update
    */
-  private static class UpdateMetadata extends StorageAction>> {
+  private static class UpdateMetadataAction extends StorageAction>> {
 
     @Override
     public void run(StorageService storage, Tuple> tuple)
@@ -467,6 +478,52 @@ public String params() {
     }
   }
 
+  /**
+   * This class demonstrates how to sign a url.
+   * URL will be valid for 1 day.
+   *
+   * @see Signed URLs
+   */
+  private static class SignUrlAction extends
+      StorageAction> {
+
+    private static final char[] PASSWORD =  "notasecret".toCharArray();
+
+    @Override
+    public void run(StorageService storage, Tuple tuple)
+        throws Exception {
+      run(storage, tuple.x(), tuple.y());
+    }
+
+    private void run(StorageService storage, ServiceAccountAuthCredentials cred, Blob blob)
+        throws IOException {
+      Calendar cal = Calendar.getInstance();
+      cal.add(Calendar.DATE, 1);
+      long expiration = cal.getTimeInMillis() / 1000;
+      System.out.println("Signed URL: " +
+          storage.signUrl(blob, expiration, SignUrlOption.serviceAccount(cred)));
+    }
+
+    @Override
+    Tuple parse(String... args)
+        throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException,
+        UnrecoverableKeyException {
+      if (args.length != 4) {
+        throw new IllegalArgumentException();
+      }
+      KeyStore keystore = KeyStore.getInstance("PKCS12");
+      keystore.load(Files.newInputStream(Paths.get(args[0])), PASSWORD);
+      PrivateKey privateKey = (PrivateKey) keystore.getKey("privatekey", PASSWORD);
+      ServiceAccountAuthCredentials cred = AuthCredentials.createFor(args[1], privateKey);
+      return Tuple.of(cred, Blob.of(args[2], args[3]));
+    }
+
+    @Override
+    public String params() {
+      return "   ";
+    }
+  }
+
   static {
     ACTIONS.put("info", new InfoAction());
     ACTIONS.put("delete", new DeleteAction());
@@ -475,7 +532,8 @@ public String params() {
     ACTIONS.put("download", new DownloadAction());
     ACTIONS.put("cp", new CopyAction());
     ACTIONS.put("compose", new ComposeAction());
-    ACTIONS.put("update_metadata", new UpdateMetadata());
+    ACTIONS.put("update_metadata", new UpdateMetadataAction());
+    ACTIONS.put("sign_url", new SignUrlAction());
   }
 
   public static void printUsage() {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index 12c74b811ea9..f63c57e3c784 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -81,8 +81,8 @@ public DefaultStorageRpc(StorageServiceOptions options) {
     HttpTransport transport = options.httpTransportFactory().create();
     HttpRequestInitializer initializer = options.httpRequestInitializer();
     this.options = options;
-    // todo: validate what is returned by getRootURL and use that for host default (and use host())
     storage = new Storage.Builder(transport, new JacksonFactory(), initializer)
+        .setRootUrl(options.host())
         .setApplicationName("gcloud-java")
         .build();
   }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
index 340b473cb0f3..ecf7064a503c 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
@@ -29,6 +29,7 @@
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_MATCH;
 import static com.google.gcloud.spi.StorageRpc.Option.IF_SOURCE_METAGENERATION_NOT_MATCH;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Function;
@@ -51,6 +52,7 @@
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.net.URLEncoder;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.security.Signature;
@@ -467,7 +469,7 @@ public URL signUrl(Blob blob, long expiration, SignUrlOption... options) {
       stBuilder.append(blob.contentType());
     }
     stBuilder.append('\n');
-    stBuilder.append(expiration).append('\n').append('\n');
+    stBuilder.append(expiration).append('\n');
     StringBuilder path = new StringBuilder();
     if (!blob.bucket().startsWith("/")) {
       path.append('/');
@@ -484,9 +486,9 @@ public URL signUrl(Blob blob, long expiration, SignUrlOption... options) {
     try {
       Signature signer = Signature.getInstance("SHA256withRSA");
       signer.initSign(cred.privateKey());
-      signer.update(stBuilder.toString().getBytes("UTF-8"));
-      String signature = BaseEncoding.base64Url().encode(signer.sign());
-      // todo - use options().host() - after default is correct and value is past to RPC
+      signer.update(stBuilder.toString().getBytes(UTF_8));
+      String signature =
+          URLEncoder.encode(BaseEncoding.base64().encode(signer.sign()), UTF_8.name());
       stBuilder = new StringBuilder("https://storage.googleapis.com").append(path);
       stBuilder.append("?GoogleAccessId=").append(cred.account());
       stBuilder.append("&Expires=").append(expiration);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java
index f29020c6380f..f978cb87f3d1 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CorsTest.java
@@ -38,7 +38,7 @@ public void testOrigin() {
   public void corsTest() {
     List origins = ImmutableList.of(Origin.any(), Origin.of("o"));
     List headers = ImmutableList.of("h1", "h2");
-    List methods = ImmutableList.of(HttpMethod.ANY);
+    List methods = ImmutableList.of(HttpMethod.GET);
     Cors cors = Cors.builder()
         .maxAgeSeconds(100)
         .origins(origins)
diff --git a/pom.xml b/pom.xml
index e6fafbe5e7c2..8f56620542bc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,6 +85,19 @@
     
   
   
+    
+      
+        
+          org.codehaus.mojo
+          exec-maven-plugin
+          1.3.2
+          
+            true
+            java
+          
+        
+      
+    
     
       
         org.codehaus.mojo

From 2891eac8e5749ab81b758fa6a4b2fb203b3eeeed Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 28 May 2015 20:05:33 -0700
Subject: [PATCH 252/732] enforce minium Java and Maven requirements

---
 pom.xml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/pom.xml b/pom.xml
index e6fafbe5e7c2..0eaf0a920fa3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -86,6 +86,29 @@
   
   
     
+      
+        org.apache.maven.plugins
+        maven-enforcer-plugin
+        1.4
+        
+          
+            enforce-maven
+            
+              enforce
+            
+            
+              
+                
+                  [3.0,)
+                
+                
+                  [1.7,)
+                
+                  
+            
+          
+        
+      
       
         org.codehaus.mojo
         exec-maven-plugin

From 378b006eac22e9b5dc364f8442cd47c871ed4cc0 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Thu, 28 May 2015 20:30:21 -0700
Subject: [PATCH 253/732] rename XXXService, XXXServiceFactory,
 XXXServiceException and XXXServiceOptions to XXX, XXXFactory, XXXException
 and XXXOptions

---
 README.md                                     | 10 +--
 gcloud-java-datastore/README.md               | 10 +--
 .../datastore/BaseDatastoreBatchWriter.java   |  6 +-
 .../google/gcloud/datastore/BaseEntity.java   | 24 +++----
 .../com/google/gcloud/datastore/Batch.java    |  6 +-
 .../google/gcloud/datastore/BatchImpl.java    |  6 +-
 .../{DatastoreService.java => Datastore.java} | 20 +++---
 .../datastore/DatastoreBatchWriter.java       | 10 +--
 ...Exception.java => DatastoreException.java} | 34 ++++-----
 ...viceFactory.java => DatastoreFactory.java} | 16 ++---
 .../gcloud/datastore/DatastoreHelper.java     | 12 ++--
 ...oreServiceImpl.java => DatastoreImpl.java} | 20 +++---
 ...viceOptions.java => DatastoreOptions.java} | 24 +++----
 .../gcloud/datastore/DatastoreReader.java     |  8 +--
 .../gcloud/datastore/DatastoreWriter.java     |  4 +-
 .../google/gcloud/datastore/KeyFactory.java   |  2 +-
 .../google/gcloud/datastore/ListValue.java    |  2 +-
 .../google/gcloud/datastore/QueryResults.java |  2 +-
 .../gcloud/datastore/QueryResultsImpl.java    |  4 +-
 .../google/gcloud/datastore/Transaction.java  | 16 ++---
 .../gcloud/datastore/TransactionImpl.java     |  6 +-
 .../google/gcloud/datastore/package-info.java |  5 +-
 .../gcloud/spi/DatastoreRpcFactory.java       |  4 +-
 .../gcloud/spi/DefaultDatastoreRpc.java       |  4 +-
 .../BaseDatastoreBatchWriterTest.java         |  2 +-
 .../gcloud/datastore/BaseEntityTest.java      |  2 +-
 .../datastore/DatastoreExceptionTest.java     |  1 +
 .../gcloud/datastore/DatastoreHelperTest.java | 72 +++++++++----------
 ...onsTest.java => DatastoreOptionsTest.java} | 12 ++--
 .../DatastoreServiceExceptionTest.java        |  1 -
 ...oreServiceTest.java => DatastoreTest.java} | 46 ++++++------
 .../gcloud/datastore/ListValueTest.java       |  2 +-
 .../gcloud/datastore/SerializationTest.java   |  4 +-
 .../gcloud/examples/DatastoreExample.java     | 14 ++--
 .../gcloud/examples/StorageExample.java       | 40 +++++------
 .../google/gcloud/spi/DefaultStorageRpc.java  | 58 +++++++--------
 .../com/google/gcloud/spi/StorageRpc.java     |  2 +-
 .../google/gcloud/spi/StorageRpcFactory.java  |  4 +-
 .../google/gcloud/storage/BatchRequest.java   |  4 +-
 .../google/gcloud/storage/BatchResponse.java  | 10 +--
 .../gcloud/storage/BlobReadChannelImpl.java   |  6 +-
 .../gcloud/storage/BlobWriterChannelImpl.java |  8 +--
 .../{StorageService.java => Storage.java}     | 34 ++++-----
 ...ceException.java => StorageException.java} |  2 +-
 ...erviceFactory.java => StorageFactory.java} | 16 ++---
 ...orageServiceImpl.java => StorageImpl.java} | 30 ++++----
 ...erviceOptions.java => StorageOptions.java} | 18 ++---
 .../google/gcloud/storage/package-info.java   |  4 +-
 .../gcloud/storage/BatchRequestTest.java      |  6 +-
 .../gcloud/storage/SerializationTest.java     | 28 ++++----
 50 files changed, 340 insertions(+), 341 deletions(-)
 rename gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/{DatastoreService.java => Datastore.java} (80%)
 rename gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/{DatastoreServiceException.java => DatastoreException.java} (76%)
 rename gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/{DatastoreServiceFactory.java => DatastoreFactory.java} (61%)
 rename gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/{DatastoreServiceImpl.java => DatastoreImpl.java} (95%)
 rename gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/{DatastoreServiceOptions.java => DatastoreOptions.java} (88%)
 create mode 100644 gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java
 rename gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/{DatastoreServiceOptionsTest.java => DatastoreOptionsTest.java} (90%)
 delete mode 100644 gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
 rename gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/{DatastoreServiceTest.java => DatastoreTest.java} (95%)
 rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{StorageService.java => Storage.java} (94%)
 rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{StorageServiceException.java => StorageException.java} (61%)
 rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{StorageServiceFactory.java => StorageFactory.java} (62%)
 rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{StorageServiceImpl.java => StorageImpl.java} (94%)
 rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{StorageServiceOptions.java => StorageOptions.java} (83%)

diff --git a/README.md b/README.md
index 68c9db99c743..2baf37e639ad 100644
--- a/README.md
+++ b/README.md
@@ -56,16 +56,16 @@ See the ``gcloud-java`` API [datastore documentation][datastore-api] to learn ho
 with the Cloud Datastore using this Client Library.
 
 ```java
-import com.google.gcloud.datastore.DatastoreService;
-import com.google.gcloud.datastore.DatastoreServiceFactory;
-import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.Datastore;
+import com.google.gcloud.datastore.DatastoreFactory;
+import com.google.gcloud.datastore.DatastoreOptions;
 import com.google.gcloud.datastore.DateTime;
 import com.google.gcloud.datastore.Entity;
 import com.google.gcloud.datastore.Key;
 import com.google.gcloud.datastore.KeyFactory;
 
-DatastoreServiceOptions options = DatastoreServiceOptions.builder().projectId(PROJECT_ID).build();
-DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
+DatastoreOptions options = DatastoreOptions.builder().projectId(PROJECT_ID).build();
+Datastore datastore = DatastoreFactory.instance().get(options);
 KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
 Key key = keyFactory.newKey(keyName);
 Entity entity = datastore.get(key);
diff --git a/gcloud-java-datastore/README.md b/gcloud-java-datastore/README.md
index 96d8075706e9..7113dbdd0231 100644
--- a/gcloud-java-datastore/README.md
+++ b/gcloud-java-datastore/README.md
@@ -40,16 +40,16 @@ See the ``gcloud-java`` API [datastore documentation][datastore-api] to learn ho
 with the Cloud Datastore using this Client Library.
 
 ```java
-import com.google.gcloud.datastore.DatastoreService;
-import com.google.gcloud.datastore.DatastoreServiceFactory;
-import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.Datastore;
+import com.google.gcloud.datastore.DatastoreFactory;
+import com.google.gcloud.datastore.DatastoreOptions;
 import com.google.gcloud.datastore.DateTime;
 import com.google.gcloud.datastore.Entity;
 import com.google.gcloud.datastore.Key;
 import com.google.gcloud.datastore.KeyFactory;
 
-DatastoreServiceOptions options = DatastoreServiceOptions.builder().projectId(PROJECT_ID).build();
-DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
+DatastoreOptions options = DatastoreOptions.builder().projectId(PROJECT_ID).build();
+Datastore datastore = DatastoreFactory.instance().get(options);
 KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
 Key key = keyFactory.newKey(keyName);
 Entity entity = datastore.get(key);
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
index 655b1a32125c..7eaf5c535f26 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseDatastoreBatchWriter.java
@@ -195,8 +195,8 @@ protected void validateActive() {
     }
   }
 
-  protected DatastoreServiceException newInvalidRequest(String msg, Object... params) {
-    return DatastoreServiceException.throwInvalidRequest(String.format(msg, params));
+  protected DatastoreException newInvalidRequest(String msg, Object... params) {
+    return DatastoreException.throwInvalidRequest(String.format(msg, params));
   }
 
   protected DatastoreV1.Mutation.Builder toMutationPb() {
@@ -219,5 +219,5 @@ protected DatastoreV1.Mutation.Builder toMutationPb() {
     return mutationPb;
   }
 
-  protected abstract DatastoreService datastore();
+  protected abstract Datastore datastore();
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 74417f9bc756..21e20b33a8ed 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -242,13 +242,13 @@ public boolean contains(String name) {
   /**
    * Returns the {@link Value} for the given property {@code name}.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    */
   public > V getValue(String name) {
     @SuppressWarnings("unchecked")
     V property = (V) properties.get(name);
     if (property == null) {
-      throw DatastoreServiceException.throwInvalidRequest("No such property %s", name);
+      throw DatastoreException.throwInvalidRequest("No such property %s", name);
     }
     return property;
   }
@@ -256,7 +256,7 @@ public > V getValue(String name) {
   /**
    * Returns true if property is instanceof NullValue.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    */
   public boolean isNull(String name) {
     return getValue(name) instanceof NullValue;
@@ -266,7 +266,7 @@ public boolean isNull(String name) {
   /**
    * Returns the property value as a string.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a string.
    */
   @SuppressWarnings("unchecked")
@@ -277,7 +277,7 @@ public String getString(String name) {
   /**
    * Returns the property value as long.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a long.
    */
   @SuppressWarnings("unchecked")
@@ -288,7 +288,7 @@ public long getLong(String name) {
   /**
    * Returns the property value as a double.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a double.
    */
   @SuppressWarnings("unchecked")
@@ -299,7 +299,7 @@ public double getDouble(String name) {
   /**
    * Returns the property value as a boolean.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a boolean.
    */
   @SuppressWarnings("unchecked")
@@ -310,7 +310,7 @@ public boolean getBoolean(String name) {
   /**
    * Returns the property value as a DateTime.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a DateTime.
    */
   @SuppressWarnings("unchecked")
@@ -321,7 +321,7 @@ public DateTime getDateTime(String name) {
   /**
    * Returns the property value as a Key.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a Key.
    */
   @SuppressWarnings("unchecked")
@@ -332,7 +332,7 @@ public Key getKey(String name) {
   /**
    * Returns the property value as an entity.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not an entity.
    */
   @SuppressWarnings("unchecked")
@@ -343,7 +343,7 @@ public  FullEntity getEntity(String name) {
   /**
    * Returns the property value as a list of values.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a list of values.
    */
   @SuppressWarnings("unchecked")
@@ -354,7 +354,7 @@ public List> getList(String name) {
   /**
    * Returns the property value as a blob.
    *
-   * @throws DatastoreServiceException if not such property.
+   * @throws DatastoreException if not such property.
    * @throws ClassCastException if value is not a blob.
    */
   @SuppressWarnings("unchecked")
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Batch.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Batch.java
index f74ccc288808..75a5d1381403 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Batch.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Batch.java
@@ -43,12 +43,12 @@ interface Response {
   /**
    * Submit the batch to the Datastore.
    *
-   * @throws DatastoreServiceException if there was any failure or if batch is not longer active
+   * @throws DatastoreException if there was any failure or if batch is not longer active
    */
   Response submit();
 
   /**
-   * Returns the batch associated {@link DatastoreService}.
+   * Returns the batch associated {@link Datastore}.
    */
-  DatastoreService datastore();
+  Datastore datastore();
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
index 0d7eb704d1eb..9c95949e2c8a 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -27,7 +27,7 @@
 
 class BatchImpl extends BaseDatastoreBatchWriter implements Batch {
 
-  private final DatastoreServiceImpl datastore;
+  private final DatastoreImpl datastore;
   private final boolean force;
 
   static class ResponseImpl implements Batch.Response {
@@ -49,7 +49,7 @@ public List generatedKeys() {
     }
   }
 
-  BatchImpl(DatastoreServiceImpl datastore, BatchOption... options) {
+  BatchImpl(DatastoreImpl datastore, BatchOption... options) {
     super("batch");
     this.datastore = datastore;
     Map, BatchOption> optionsMap = BatchOption.asImmutableMap(options);
@@ -76,7 +76,7 @@ public Batch.Response submit() {
   }
 
   @Override
-  public DatastoreService datastore() {
+  public Datastore datastore() {
     return datastore;
   }
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreService.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java
similarity index 80%
rename from gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreService.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java
index 7d27938fd246..fe79fdf45ff4 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreService.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java
@@ -23,12 +23,12 @@
 /**
  * An interface for Google Cloud Datastore.
  */
-public interface DatastoreService extends Service, DatastoreReaderWriter {
+public interface Datastore extends Service, DatastoreReaderWriter {
 
   /**
    * Returns a new Datastore transaction.
    *
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    */
   Transaction newTransaction(TransactionOption... options);
 
@@ -47,15 +47,15 @@ interface TransactionCallable {
 
 
   /**
-   * Invokes the callback's {@link DatastoreService.TransactionCallable#run} method with a
+   * Invokes the callback's {@link Datastore.TransactionCallable#run} method with a
    * {@link DatastoreReaderWriter} that is associated with a new transaction.
    * The transaction will be committed upon successful invocation.
    * Any thrown exception will cause the transaction to rollback and will be propagated
-   * as a {@link DatastoreServiceException} with the original exception as its root cause.
+   * as a {@link DatastoreException} with the original exception as its root cause.
    *
    * @param callable the callback to call with a newly created transactional readerWriter
    * @param options the options for the created transaction
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    */
    T runInTransaction(TransactionCallable callable, TransactionOption... options);
 
@@ -69,35 +69,35 @@ interface TransactionCallable {
    * The returned key will have the same information (projectId, kind, namespace and ancestors)
    * as the given key and will have a newly assigned id.
    *
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    */
   Key allocateId(IncompleteKey key);
 
   /**
    * Returns a list of keys using the allocated ids ordered by the input.
    *
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    * @see #allocateId(IncompleteKey)
    */
   List allocateId(IncompleteKey... key);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    */
   @Override
   void update(Entity... entity);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    */
   @Override
   void put(Entity... entity);
 
   /**
    * {@inheritDoc}
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    */
   @Override
   void delete(Key... key);
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
index 7e1ee6d57128..3a80452349dc 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreBatchWriter.java
@@ -31,7 +31,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    * to submit time.
    *
    * @throws IllegalArgumentException if any of the given entities is missing a key
-   * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity with a
+   * @throws DatastoreException if a given entity with a
    *     complete key was already added to this writer or if not active
    */
   void addWithDeferredIdAllocation(FullEntity... entity);
@@ -40,7 +40,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    * {@inheritDoc}
    * For entities with complete keys that were marked for deletion in this writer the operation
    * will be changed to {@link #put}.
-   * @throws com.google.gcloud.datastore.DatastoreServiceException if a given entity with the
+   * @throws DatastoreException if a given entity with the
    *     same complete key was already added to this writer, if writer is not active or
    *     if id allocation for an entity with an incomplete key failed.
    */
@@ -51,7 +51,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    * {@inheritDoc}
    * This operation will be converted to {@link #put} operation for entities that were already
    *     added or put in this writer
-   * @throws com.google.gcloud.datastore.DatastoreServiceException if an entity is marked for
+   * @throws DatastoreException if an entity is marked for
    *     deletion in this writer or if not active
    */
   @Override
@@ -61,7 +61,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
    * {@inheritDoc}
    * This operation will also remove from this batch any prior writes for entities with the same
    *     keys
-   * @throws com.google.gcloud.datastore.DatastoreServiceException if not active
+   * @throws DatastoreException if not active
    */
   @Override
   void delete(Key... key);
@@ -69,7 +69,7 @@ interface DatastoreBatchWriter extends DatastoreWriter {
   /**
    * {@inheritDoc}
    * This operation will also remove from this writer any prior writes for the same entities.
-   * @throws com.google.gcloud.datastore.DatastoreServiceException if not active
+   * @throws DatastoreException if not active
    */
   @Override
   void put(Entity... entity);
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
similarity index 76%
rename from gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
index 0d958005c22a..d91cc2ccd98b 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceException.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
@@ -26,7 +26,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-public class DatastoreServiceException extends RuntimeException {
+public class DatastoreException extends RuntimeException {
 
   private static final long serialVersionUID = 8170357898917041899L;
   private static final ImmutableMap REASON_TO_CODE;
@@ -82,8 +82,8 @@ public boolean retryable() {
       return retryable;
     }
 
-    DatastoreServiceException translate(DatastoreRpcException exception, String message) {
-      return new DatastoreServiceException(this, message, exception);
+    DatastoreException translate(DatastoreRpcException exception, String message) {
+      return new DatastoreException(this, message, exception);
     }
   }
 
@@ -98,12 +98,12 @@ DatastoreServiceException translate(DatastoreRpcException exception, String mess
     HTTP_TO_CODE = ImmutableMap.copyOf(httpCodes);
   }
 
-  public DatastoreServiceException(Code code, String message, Exception cause) {
+  public DatastoreException(Code code, String message, Exception cause) {
     super(MoreObjects.firstNonNull(message, code.description), cause);
     this.code = code;
   }
 
-  public DatastoreServiceException(Code code, String message) {
+  public DatastoreException(Code code, String message) {
     this(code, message, null);
   }
 
@@ -114,23 +114,23 @@ public Code code() {
     return code;
   }
 
-  static DatastoreServiceException translateAndThrow(RetryHelperException ex) {
+  static DatastoreException translateAndThrow(RetryHelperException ex) {
     if (ex.getCause() instanceof DatastoreRpcException) {
       return translateAndThrow((DatastoreRpcException) ex.getCause());
     }
     if (ex instanceof RetryHelper.RetryInterruptedException) {
       RetryHelper.RetryInterruptedException.propagate();
     }
-    throw new DatastoreServiceException(Code.UNKNOWN, ex.getMessage(), ex);
+    throw new DatastoreException(Code.UNKNOWN, ex.getMessage(), ex);
   }
 
   /**
-   * Translate DatastoreException to DatastoreServiceException based on their
-   * HTTP error codes. This method will always throw a new DatastoreServiceException.
+   * Translate DatastoreException to DatastoreException based on their
+   * HTTP error codes. This method will always throw a new DatastoreException.
    *
-   * @throws DatastoreServiceException every time
+   * @throws DatastoreException every time
    */
-  static DatastoreServiceException translateAndThrow(DatastoreRpcException exception) {
+  static DatastoreException translateAndThrow(DatastoreRpcException exception) {
     String message = exception.getMessage();
     Code code = REASON_TO_CODE.get(exception.reason());
     if (code == null) {
@@ -140,16 +140,16 @@ static DatastoreServiceException translateAndThrow(DatastoreRpcException excepti
   }
 
   /**
-   * Throw a DatastoreServiceException with {@code FAILED_PRECONDITION} code and the {@code message}
+   * Throw a DatastoreException with {@code FAILED_PRECONDITION} code and the {@code message}
    * in a nested exception.
    *
-   * @throws DatastoreServiceException every time
+   * @throws DatastoreException every time
    */
-  static DatastoreServiceException throwInvalidRequest(String massage, Object... params) {
-    throw new DatastoreServiceException(Code.FAILED_PRECONDITION, String.format(massage, params));
+  static DatastoreException throwInvalidRequest(String massage, Object... params) {
+    throw new DatastoreException(Code.FAILED_PRECONDITION, String.format(massage, params));
   }
 
-  static DatastoreServiceException propagateUserException(Exception ex) {
-    throw new DatastoreServiceException(Code.UNKNOWN, ex.getMessage(), ex);
+  static DatastoreException propagateUserException(Exception ex) {
+    throw new DatastoreException(Code.UNKNOWN, ex.getMessage(), ex);
   }
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreFactory.java
similarity index 61%
rename from gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreFactory.java
index ce0887792f98..a64fab3715f1 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceFactory.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreFactory.java
@@ -18,26 +18,26 @@
 
 
 /**
- * A base class for DatasoreService factories.
+ * A base class for Datastore factories.
  */
-public abstract class DatastoreServiceFactory {
+public abstract class DatastoreFactory {
 
-  private static final DatastoreServiceFactory INSTANCE = new DatastoreServiceFactory() {
+  private static final DatastoreFactory INSTANCE = new DatastoreFactory() {
       @Override
-      public DatastoreService get(DatastoreServiceOptions options) {
-        return new DatastoreServiceImpl(options);
+      public Datastore get(DatastoreOptions options) {
+        return new DatastoreImpl(options);
       }
     };
 
   /**
    * Returns the default factory instance.
    */
-  public static DatastoreServiceFactory instance() {
+  public static DatastoreFactory instance() {
     return INSTANCE;
   }
 
   /**
-   * Returns a {@code DatastoreService} for the given options.
+   * Returns a {@code Datastore} service for the given options.
    */
-  public abstract DatastoreService get(DatastoreServiceOptions options);
+  public abstract Datastore get(DatastoreOptions options);
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
index d07db9ece9b2..a74d06642740 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreHelper.java
@@ -34,7 +34,7 @@ private DatastoreHelper() {
   }
 
 
-  static Key allocateId(DatastoreService service, IncompleteKey key) {
+  static Key allocateId(Datastore service, IncompleteKey key) {
     return service.allocateId(new IncompleteKey[]{key}).get(0);
   }
 
@@ -46,7 +46,7 @@ static Entity add(DatastoreWriter writer, FullEntity entity) {
     return writer.add(new FullEntity[] {entity}).get(0);
   }
 
-  static KeyFactory newKeyFactory(DatastoreServiceOptions options) {
+  static KeyFactory newKeyFactory(DatastoreOptions options) {
     return new KeyFactory(options.projectId(), options.namespace());
   }
 
@@ -69,16 +69,16 @@ static List fetch(DatastoreReader reader, Key... keys) {
     return list;
   }
 
-  static  T runInTransaction(DatastoreService datastoreService,
-      DatastoreService.TransactionCallable callable, TransactionOption... options) {
-    Transaction transaction = datastoreService.newTransaction(options);
+  static  T runInTransaction(Datastore datastore,
+      Datastore.TransactionCallable callable, TransactionOption... options) {
+    Transaction transaction = datastore.newTransaction(options);
     try {
       T value = callable.run(transaction);
       transaction.commit();
       return value;
     } catch (Exception ex) {
       transaction.rollback();
-      throw DatastoreServiceException.propagateUserException(ex);
+      throw DatastoreException.propagateUserException(ex);
     } finally {
       if (transaction.active()) {
         transaction.rollback();
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java
similarity index 95%
rename from gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java
index d7712dd0bf15..e848dd5e56c8 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java
@@ -45,8 +45,8 @@
 import java.util.concurrent.Callable;
 
 
-final class DatastoreServiceImpl extends BaseService
-    implements DatastoreService {
+final class DatastoreImpl extends BaseService
+    implements Datastore {
 
   private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
       new Interceptor() {
@@ -74,7 +74,7 @@ public RetryResult beforeEval(Exception exception) {
   private final DatastoreRpc datastoreRpc;
   private final RetryParams retryParams;
 
-  DatastoreServiceImpl(DatastoreServiceOptions options) {
+  DatastoreImpl(DatastoreOptions options) {
     super(options);
     this.datastoreRpc = options.datastoreRpc();
     retryParams = MoreObjects.firstNonNull(options.retryParams(), RetryParams.noRetries());
@@ -112,7 +112,7 @@ DatastoreV1.RunQueryResponse runQuery(final DatastoreV1.RunQueryRequest requestP
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+      throw DatastoreException.translateAndThrow(e);
     }
   }
 
@@ -150,7 +150,7 @@ DatastoreV1.AllocateIdsResponse allocateIds(final DatastoreV1.AllocateIdsRequest
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+      throw DatastoreException.translateAndThrow(e);
     }
   }
 
@@ -181,7 +181,7 @@ public List add(FullEntity... entities) {
       }
       if (completeEntity != null) {
         if (completeEntities.put(completeEntity.key(), completeEntity) != null) {
-          throw DatastoreServiceException.throwInvalidRequest(
+          throw DatastoreException.throwInvalidRequest(
               "Duplicate entity with the key %s", entity.key());
         }
         mutationPb.addInsert(completeEntity.toPb());
@@ -277,7 +277,7 @@ DatastoreV1.LookupResponse lookup(final DatastoreV1.LookupRequest requestPb) {
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+      throw DatastoreException.translateAndThrow(e);
     }
   }
 
@@ -348,7 +348,7 @@ DatastoreV1.CommitResponse commit(final DatastoreV1.CommitRequest requestPb) {
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+      throw DatastoreException.translateAndThrow(e);
     }
   }
 
@@ -366,7 +366,7 @@ public DatastoreV1.BeginTransactionResponse call() throws DatastoreRpcException
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+      throw DatastoreException.translateAndThrow(e);
     }
   }
 
@@ -385,7 +385,7 @@ void rollback(final DatastoreV1.RollbackRequest requestPb) {
         }
       }, retryParams, EXCEPTION_HANDLER);
     } catch (RetryHelperException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+      throw DatastoreException.translateAndThrow(e);
     }
   }
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
similarity index 88%
rename from gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
rename to gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
index e0dba7b7a982..ed6b51458938 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreServiceOptions.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
@@ -34,7 +34,7 @@
 import java.util.Objects;
 import java.util.Set;
 
-public class DatastoreServiceOptions extends ServiceOptions {
+public class DatastoreOptions extends ServiceOptions {
 
   private static final long serialVersionUID = -8636602944160689193L;
   private static final String DATASET_ENV_NAME = "DATASTORE_DATASET";
@@ -48,7 +48,7 @@ public class DatastoreServiceOptions extends ServiceOptions {
+      ServiceOptions.Builder {
 
     private String namespace;
     private boolean force;
@@ -57,7 +57,7 @@ public static class Builder extends
     private Builder() {
     }
 
-    private Builder(DatastoreServiceOptions options) {
+    private Builder(DatastoreOptions options) {
       super(options);
       force = options.force;
       namespace = options.namespace;
@@ -65,8 +65,8 @@ private Builder(DatastoreServiceOptions options) {
     }
 
     @Override
-    public DatastoreServiceOptions build() {
-      DatastoreServiceOptions options = new DatastoreServiceOptions(this);
+    public DatastoreOptions build() {
+      DatastoreOptions options = new DatastoreOptions(this);
       return normalizeDataset ? options.normalize() : options;
     }
 
@@ -86,14 +86,14 @@ Builder normalizeDataset(boolean normalizeDataset) {
     }
   }
 
-  private DatastoreServiceOptions(Builder builder) {
+  private DatastoreOptions(Builder builder) {
     super(builder);
     normalizeDataset = builder.normalizeDataset;
     namespace = builder.namespace != null ? builder.namespace : defaultNamespace();
     force = builder.force;
   }
 
-  private DatastoreServiceOptions normalize() {
+  private DatastoreOptions normalize() {
     if (!normalizeDataset) {
       return this;
     }
@@ -116,9 +116,9 @@ private DatastoreServiceOptions normalize() {
         key = combinedIter.next().getEntity().getKey();
       }
       builder.projectId(key.getPartitionId().getDatasetId());
-      return new DatastoreServiceOptions(builder);
+      return new DatastoreOptions(builder);
     } catch (DatastoreRpcException e) {
-      throw DatastoreServiceException.translateAndThrow(e);
+      throw DatastoreException.translateAndThrow(e);
     }
   }
 
@@ -170,10 +170,10 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (!(obj instanceof DatastoreServiceOptions)) {
+    if (!(obj instanceof DatastoreOptions)) {
       return false;
     }
-    DatastoreServiceOptions other = (DatastoreServiceOptions) obj;
+    DatastoreOptions other = (DatastoreOptions) obj;
     return isEquals(other) && Objects.equals(namespace, other.namespace)
         && Objects.equals(force, other.force)
         && Objects.equals(normalizeDataset, other.normalizeDataset);
@@ -194,7 +194,7 @@ DatastoreRpc datastoreRpc() {
     return datastoreRpc;
   }
 
-  public static DatastoreServiceOptions defaultInstance() {
+  public static DatastoreOptions defaultInstance() {
     return builder().build();
   }
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReader.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
index b1dc0a70d392..056895f850e3 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreReader.java
@@ -27,18 +27,18 @@ public interface DatastoreReader {
   /**
    * Returns an {@link Entity} for the given {@link Key} or {@code null} if does not exists.
    *
-   * @throws DatastoreServiceException upon failure.
+   * @throws DatastoreException upon failure.
    */
   Entity get(Key key);
 
   /**
    * Returns an {@link Entity} for each given {@link Key} that exists in the Datastore.
    * The order of the result is unspecified.
-   * Results are loaded lazily therefore it is possible to get a {@code DatastoreServiceException}
+   * Results are loaded lazily therefore it is possible to get a {@code DatastoreException}
    * from the returned {@code Iterator}'s {@link Iterator#hasNext hasNext} or
    * {@link Iterator#next next} methods.
    *
-   * @throws DatastoreServiceException upon failure.
+   * @throws DatastoreException upon failure.
    * @see #get(Key)
    */
   Iterator get(Key... key);
@@ -53,7 +53,7 @@ public interface DatastoreReader {
   /**
    * Submit a {@link Query} and returns its result.
    *
-   * @throws DatastoreServiceException upon failure.
+   * @throws DatastoreException upon failure.
    */
    QueryResults run(Query query);
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
index 806394552419..66ba98aed9e9 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreWriter.java
@@ -30,7 +30,7 @@ public interface DatastoreWriter {
    * @param entity the entity to add
    * @return an {@code Entity} with the same properties and a key that is either newly allocated
    *     or the same one if key is already complete
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    * @throws IllegalArgumentException if the given entity is missing a key
    */
   Entity add(FullEntity entity);
@@ -41,7 +41,7 @@ public interface DatastoreWriter {
    *
    * @return a list of {@code Entity} ordered by input with the same properties and a key that
    *     is either newly allocated or the same one if was already complete
-   * @throws DatastoreServiceException upon failure
+   * @throws DatastoreException upon failure
    * @throws IllegalArgumentException if any of the given entities is missing a key
    * @see #add(FullEntity)
    */
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index fc288d46510f..8010468c5068 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -19,7 +19,7 @@
 import com.google.common.collect.ImmutableList;
 
 /**
- * An helper for creating keys for a specific {@link DatastoreService},
+ * An helper for creating keys for a specific {@link Datastore},
  * using its associated projectId and namespace.
  */
 public final class KeyFactory extends BaseKey.Builder {
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ListValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ListValue.java
index a13dbb17cc69..41c7e82788b5 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ListValue.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ListValue.java
@@ -88,7 +88,7 @@ public Builder addValue(Value first, Value... other) {
     @Override
     public Builder indexed(boolean indexed) {
       // see issue #26
-      throw DatastoreServiceException.throwInvalidRequest("ListValue can't specify index");
+      throw DatastoreException.throwInvalidRequest("ListValue can't specify index");
     }
 
     /**
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
index 6310eec93215..44360987b573 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
@@ -22,7 +22,7 @@
  * The result of a Google Cloud Datastore query submission.
  * When result is not typed it is possible to cast it to its appropriate type according to
  * the {@link #resultClass} value.
- * Results are loaded lazily therefore it is possible to get a {@code DatastoreServiceException}
+ * Results are loaded lazily therefore it is possible to get a {@code DatastoreException}
  * upon {@link Iterator#hasNext hasNext} or {@link Iterator#next next} calls.
  *
  * @param  the type of the results value.
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
index f97c9483e002..8e2f294ed15d 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
@@ -27,7 +27,7 @@
 
 class QueryResultsImpl extends AbstractIterator implements QueryResults {
 
-  private final DatastoreServiceImpl datastore;
+  private final DatastoreImpl datastore;
   private final DatastoreV1.ReadOptions readOptionsPb;
   private final DatastoreV1.PartitionId partitionIdPb;
   private final ResultType queryResultType;
@@ -39,7 +39,7 @@ class QueryResultsImpl extends AbstractIterator implements QueryResults
   //private ByteString cursor; // only available in v1beta3
 
 
-  QueryResultsImpl(DatastoreServiceImpl datastore, DatastoreV1.ReadOptions readOptionsPb,
+  QueryResultsImpl(DatastoreImpl datastore, DatastoreV1.ReadOptions readOptionsPb,
                    Query query) {
     this.datastore = datastore;
     this.readOptionsPb = readOptionsPb;
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Transaction.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Transaction.java
index 6ac1da78ab75..9d676bc68a8c 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Transaction.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Transaction.java
@@ -62,7 +62,7 @@ interface Response {
    * to fail if entity was changed by others after it was seen by this transaction) but any
    * write changes in this transaction will not be reflected by the returned entity.
    *
-   * @throws DatastoreServiceException upon failure or if no longer active
+   * @throws DatastoreException upon failure or if no longer active
    */
   @Override
   Entity get(Key key);
@@ -73,7 +73,7 @@ interface Response {
    * to fail if any of the entities was changed by others after they were seen by this transaction)
    * but any write changes in this transaction will not be reflected by the returned entities.
    *
-   * @throws DatastoreServiceException upon failure or if no longer active
+   * @throws DatastoreException upon failure or if no longer active
    */
   @Override
   Iterator get(Key... key);
@@ -84,7 +84,7 @@ interface Response {
    * to fail if any of the entities was changed by others after they were seen by this transaction)
    * but any write changes in this transaction will not be reflected by the returned entities.
    *
-   * @throws DatastoreServiceException upon failure or if no longer active
+   * @throws DatastoreException upon failure or if no longer active
    */
   @Override
   List fetch(Key... keys);
@@ -96,7 +96,7 @@ interface Response {
    * query was performed) but any write changes in this transaction will not be reflected by
    * the result.
    *
-   * @throws DatastoreServiceException upon failure or if no longer active
+   * @throws DatastoreException upon failure or if no longer active
    */
   @Override
    QueryResults run(Query query);
@@ -104,14 +104,14 @@ interface Response {
   /**
    * Commit the transaction.
    *
-   * @throws DatastoreServiceException if could not commit the transaction or if no longer active
+   * @throws DatastoreException if could not commit the transaction or if no longer active
    */
   Response commit();
 
   /**
    * Rollback the transaction.
    *
-   * @throws DatastoreServiceException if transaction was already committed
+   * @throws DatastoreException if transaction was already committed
    */
   void rollback();
 
@@ -122,7 +122,7 @@ interface Response {
   boolean active();
 
   /**
-   * Returns the transaction associated {@link DatastoreService}.
+   * Returns the transaction associated {@link Datastore}.
    */
-  DatastoreService datastore();
+  Datastore datastore();
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 6f1090945ee1..48568650910d 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -29,7 +29,7 @@
 
 final class TransactionImpl extends BaseDatastoreBatchWriter implements Transaction {
 
-  private final DatastoreServiceImpl datastore;
+  private final DatastoreImpl datastore;
   private final ByteString transaction;
   private final boolean force;
   private boolean rolledback;
@@ -53,7 +53,7 @@ public List generatedKeys() {
     }
   }
 
-  TransactionImpl(DatastoreServiceImpl datastore, TransactionOption... options) {
+  TransactionImpl(DatastoreImpl datastore, TransactionOption... options) {
     super("transaction");
     this.datastore = datastore;
     DatastoreV1.BeginTransactionRequest.Builder requestPb =
@@ -124,7 +124,7 @@ public void rollback() {
   }
 
   @Override
-  public DatastoreService datastore() {
+  public Datastore datastore() {
     return datastore;
   }
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java
index cf32c5bfa2c2..3b402820e663 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/package-info.java
@@ -19,9 +19,8 @@
  *
  * 
    A simple usage example:
  * 
     {@code
- * DatastoreServiceOptions options =
- *     DatastoreServiceOptions.builder().projectId(PROJECT_ID).build();
- * DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
+ * DatastoreOptions options = DatastoreOptions.builder().projectId(PROJECT_ID).build();
+ * Datastore datastore = DatastoreFactory.instance().get(options);
  * KeyFactory keyFactory = datastore.newKeyFactory().kind(kind);
  * Key key = keyFactory.newKey(keyName);
  * Entity entity = datastore.get(key);
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
index 952114788f01..1815dda30f5d 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpcFactory.java
@@ -16,13 +16,13 @@
 
 package com.google.gcloud.spi;
 
-import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.DatastoreOptions;
 
 /**
  * An interface for Datastore RPC factory.
  * Implementation will be loaded via {@link java.util.ServiceLoader}.
  */
 public interface DatastoreRpcFactory extends
-    ServiceRpcFactory {
+    ServiceRpcFactory {
 }
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
index 3d909d368bc0..2f245260b325 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DefaultDatastoreRpc.java
@@ -33,7 +33,7 @@
 import com.google.api.services.datastore.client.DatastoreFactory;
 import com.google.api.services.datastore.client.DatastoreOptions.Builder;
 import com.google.common.collect.ImmutableMap;
-import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.DatastoreOptions;
 import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
 
 import org.json.JSONException;
@@ -61,7 +61,7 @@ public class DefaultDatastoreRpc implements DatastoreRpc {
     HTTP_STATUS_TO_REASON = ImmutableMap.copyOf(httpCodes);
   }
 
-  public DefaultDatastoreRpc(DatastoreServiceOptions options) {
+  public DefaultDatastoreRpc(DatastoreOptions options) {
     client = DatastoreFactory.get().create(
         new Builder()
             .dataset(options.projectId())
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
index 71576036fe3b..c3f1bfbd5a71 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.List;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder(KEY1, 1).build();
  private static final Key KEY3 = Key.builder(KEY1, 2).build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder(KEY1).build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final FullEntity INCOMPLETE_ENTITY_1 =
      Entity.builder(INCOMPLETE_KEY).build();
  private static final FullEntity INCOMPLETE_ENTITY_2 =
      Entity.builder(INCOMPLETE_KEY).set("name", "dan").build();

  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    private final DatastoreService datastore;

    protected DatastoreBatchWriter() {
      super("test");
      datastore = EasyMock.createMock(DatastoreService.class);
      IncompleteKey[] expected = {INCOMPLETE_KEY, INCOMPLETE_KEY};
      List result = ImmutableList.of(KEY2, KEY3);
      expect(datastore.allocateId(expected)).andReturn(result).times(0, 1);
      replay(datastore);
    }

    @Override
    protected DatastoreService datastore() {
      return datastore;
    }

    void finish() {
      verify(datastore);
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @After
  public void tearDown() {
    batchWriter.finish();
  }

  @Test
  public void testAdd() throws Exception {
    Entity entity2 =
        Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(entity2.toPb())
        .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb())
        .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb())
        .build();
    List entities = batchWriter
        .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2);
    assertEquals(pb, batchWriter.toMutationPb().build());
    assertEquals(ENTITY1, entities.get(0));
    assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1));
    assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2));
    assertEquals(entity2, entities.get(3));
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddWithDeferredAllocation() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb())
        .build();
    batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testAddWithDeferredAllocationWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreServiceException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreServiceException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.List;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder(KEY1, 1).build();
  private static final Key KEY3 = Key.builder(KEY1, 2).build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder(KEY1).build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final FullEntity INCOMPLETE_ENTITY_1 =
      Entity.builder(INCOMPLETE_KEY).build();
  private static final FullEntity INCOMPLETE_ENTITY_2 =
      Entity.builder(INCOMPLETE_KEY).set("name", "dan").build();

  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    private final Datastore datastore;

    protected DatastoreBatchWriter() {
      super("test");
      datastore = EasyMock.createMock(Datastore.class);
      IncompleteKey[] expected = {INCOMPLETE_KEY, INCOMPLETE_KEY};
      List result = ImmutableList.of(KEY2, KEY3);
      expect(datastore.allocateId(expected)).andReturn(result).times(0, 1);
      replay(datastore);
    }

    @Override
    protected Datastore datastore() {
      return datastore;
    }

    void finish() {
      verify(datastore);
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @After
  public void tearDown() {
    batchWriter.finish();
  }

  @Test
  public void testAdd() throws Exception {
    Entity entity2 =
        Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(entity2.toPb())
        .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb())
        .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb())
        .build();
    List entities = batchWriter
        .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2);
    assertEquals(pb, batchWriter.toMutationPb().build());
    assertEquals(ENTITY1, entities.get(0));
    assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1));
    assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2));
    assertEquals(entity2, entities.get(3));
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddWithDeferredAllocation() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb())
        .build();
    batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testAddWithDeferredAllocationWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
index 567e795a66e7..daa0c502d4b5 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final FullEntity PARTIAL_ENTITY =
      Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override public BaseEntity build() {

      return new BaseEntity(this) {

        @Override
        protected Builder emptyBuilder() {
          return new BaseEntityTest.Builder();
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreServiceException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final FullEntity PARTIAL_ENTITY =
      Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override public BaseEntity build() {

      return new BaseEntity(this) {

        @Override
        protected Builder emptyBuilder() {
          return new BaseEntityTest.Builder();
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java
new file mode 100644
index 000000000000..1dd0f255ceca
--- /dev/null
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java
@@ -0,0 +1 @@
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

import com.google.gcloud.datastore.DatastoreException.Code;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.junit.Test;

public class DatastoreExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retryable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreException exception = new DatastoreException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
index 8a31ece583ff..55c8d0cf3ce6 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreHelperTest.java
@@ -28,7 +28,7 @@
 import static org.junit.Assert.fail;
 
 import com.google.common.collect.Iterators;
-import com.google.gcloud.datastore.DatastoreService.TransactionCallable;
+import com.google.gcloud.datastore.Datastore.TransactionCallable;
 
 import org.easymock.EasyMock;
 import org.junit.Test;
@@ -40,7 +40,7 @@ public class DatastoreHelperTest {
 
   @Test
   public void testNewKeyFactory() {
-    DatastoreServiceOptions options = createMock(DatastoreServiceOptions.class);
+    DatastoreOptions options = createMock(DatastoreOptions.class);
     expect(options.projectId()).andReturn("ds1").once();
     expect(options.namespace()).andReturn("ns1").once();
     replay(options);
@@ -55,73 +55,73 @@ public void testNewKeyFactory() {
 
   @Test
   public void testAllocateId() throws Exception {
-    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    Datastore datastore = createStrictMock(Datastore.class);
     IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
     Key key1 = Key.builder(pKey1, 1).build();
-    expect(datastoreService.allocateId(new IncompleteKey[] {pKey1}))
+    expect(datastore.allocateId(new IncompleteKey[] {pKey1}))
         .andReturn(Collections.singletonList(key1));
-    replay(datastoreService);
-    assertEquals(key1, DatastoreHelper.allocateId(datastoreService, pKey1));
-    verify(datastoreService);
+    replay(datastore);
+    assertEquals(key1, DatastoreHelper.allocateId(datastore, pKey1));
+    verify(datastore);
   }
 
   @Test
   public void testGet() throws Exception {
-    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    Datastore datastore = createStrictMock(Datastore.class);
     IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
     Key key1 = Key.builder(pKey1, 1).build();
     Entity entity1 = Entity.builder(key1).build();
     Key key2 = Key.builder(pKey1, 2).build();
-    expect(datastoreService.get(new Key[]{key1}))
+    expect(datastore.get(new Key[]{key1}))
         .andReturn(Collections.singletonList(entity1).iterator());
-    expect(datastoreService.get(new Key[]{key2}))
+    expect(datastore.get(new Key[]{key2}))
         .andReturn(Collections.emptyIterator());
-    replay(datastoreService);
-    assertEquals(entity1, DatastoreHelper.get(datastoreService, key1));
-    assertNull(DatastoreHelper.get(datastoreService, key2));
-    verify(datastoreService);
+    replay(datastore);
+    assertEquals(entity1, DatastoreHelper.get(datastore, key1));
+    assertNull(DatastoreHelper.get(datastore, key2));
+    verify(datastore);
   }
 
   @Test
   public void testAdd() throws Exception {
-    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    Datastore datastore = createStrictMock(Datastore.class);
     IncompleteKey pKey = IncompleteKey.builder("ds", "k").build();
     Key key = Key.builder(pKey, 1).build();
     Entity entity = Entity.builder(key).build();
-    expect(datastoreService.add(new Entity[]{entity}))
+    expect(datastore.add(new Entity[]{entity}))
         .andReturn(Collections.singletonList(entity));
-    replay(datastoreService);
-    assertEquals(entity, DatastoreHelper.add(datastoreService, entity));
-    verify(datastoreService);
+    replay(datastore);
+    assertEquals(entity, DatastoreHelper.add(datastore, entity));
+    verify(datastore);
   }
 
   @Test
   public void testFetch() throws Exception {
-    DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    Datastore datastore = createStrictMock(Datastore.class);
     IncompleteKey pKey1 = IncompleteKey.builder("ds", "k").build();
     Key key1 = Key.builder(pKey1, 1).build();
     Key key2 = Key.builder(pKey1, "a").build();
     Entity entity1 = Entity.builder(key1).build();
     Entity entity2 = Entity.builder(key2).build();
-    expect(datastoreService.get(key1, key2)).andReturn(Iterators.forArray(entity1, entity2)).once();
-    replay(datastoreService);
-    List values = DatastoreHelper.fetch(datastoreService, key1, key2);
+    expect(datastore.get(key1, key2)).andReturn(Iterators.forArray(entity1, entity2)).once();
+    replay(datastore);
+    List values = DatastoreHelper.fetch(datastore, key1, key2);
     assertEquals(2, values.size());
     assertEquals(entity1, values.get(0));
     assertEquals(entity2, values.get(1));
-    verify(datastoreService);
+    verify(datastore);
   }
 
   @Test
   public void testRunInTransaction() throws Exception {
-    final DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    final Datastore datastore = createStrictMock(Datastore.class);
     final Transaction transaction = createStrictMock(Transaction.class);
-    expect(datastoreService.newTransaction()).andReturn(transaction).once();
+    expect(datastore.newTransaction()).andReturn(transaction).once();
     expect(transaction.active()).andReturn(true).once();
     expect(transaction.commit()).andReturn(null).once();
     expect(transaction.active()).andReturn(false).once();
-    replay(datastoreService, transaction);
-    String value = DatastoreHelper.runInTransaction(datastoreService,
+    replay(datastore, transaction);
+    String value = DatastoreHelper.runInTransaction(datastore,
         new TransactionCallable() {
           @Override
           public String run(DatastoreReaderWriter readerWriter) {
@@ -130,22 +130,22 @@ public String run(DatastoreReaderWriter readerWriter) {
             return "done";
           }
         });
-    verify(datastoreService, transaction);
+    verify(datastore, transaction);
     assertEquals("done", value);
   }
 
   @Test
   public void testRunInTransactionWithException() throws Exception {
-    final DatastoreService datastoreService = createStrictMock(DatastoreService.class);
+    final Datastore datastore = createStrictMock(Datastore.class);
     final Transaction transaction = createStrictMock(Transaction.class);
-    expect(datastoreService.newTransaction()).andReturn(transaction).once();
+    expect(datastore.newTransaction()).andReturn(transaction).once();
     expect(transaction.active()).andReturn(true).once();
     transaction.rollback();
     EasyMock.expectLastCall().once();
     expect(transaction.active()).andReturn(false).once();
-    replay(datastoreService, transaction);
+    replay(datastore, transaction);
     try {
-      DatastoreHelper.runInTransaction(datastoreService, new TransactionCallable() {
+      DatastoreHelper.runInTransaction(datastore, new TransactionCallable() {
         @Override
         public Void run(DatastoreReaderWriter readerWriter) throws Exception {
           assertTrue(transaction.active());
@@ -153,10 +153,10 @@ public Void run(DatastoreReaderWriter readerWriter) throws Exception {
           throw new Exception("Bla");
         }
       });
-      fail("DatastoreServiceException was expected");
-    } catch (DatastoreServiceException ex) {
+      fail("DatastoreException was expected");
+    } catch (DatastoreException ex) {
       assertEquals("Bla", ex.getCause().getMessage());
     }
-    verify(datastoreService, transaction);
+    verify(datastore, transaction);
   }
 }
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java
similarity index 90%
rename from gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java
index 59468be58129..e7dc71c50ff6 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceOptionsTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreOptionsTest.java
@@ -31,23 +31,23 @@
 
 import java.io.IOException;
 
-public class DatastoreServiceOptionsTest {
+public class DatastoreOptionsTest {
 
   private static final String PROJECT_ID = "project_id";
   private DatastoreRpcFactory datastoreRpcFactory;
   private DatastoreRpc datastoreRpc;
-  private DatastoreServiceOptions.Builder options;
+  private DatastoreOptions.Builder options;
 
   @Before
   public void setUp() throws IOException, InterruptedException {
     datastoreRpcFactory = EasyMock.createMock(DatastoreRpcFactory.class);
     datastoreRpc = EasyMock.createMock(DatastoreRpc.class);
-    options = DatastoreServiceOptions.builder()
+    options = DatastoreOptions.builder()
         .normalizeDataset(false)
         .serviceRpcFactory(datastoreRpcFactory)
         .projectId(PROJECT_ID)
         .host("http://localhost:" + LocalGcdHelper.PORT);
-    EasyMock.expect(datastoreRpcFactory.create(EasyMock.anyObject(DatastoreServiceOptions.class)))
+    EasyMock.expect(datastoreRpcFactory.create(EasyMock.anyObject(DatastoreOptions.class)))
         .andReturn(datastoreRpc)
         .anyTimes();
     EasyMock.replay(datastoreRpcFactory, datastoreRpc);
@@ -83,8 +83,8 @@ public void testDatastore() throws Exception {
 
   @Test
   public void testToBuilder() throws Exception {
-    DatastoreServiceOptions original = options.namespace("ns1").force(true).build();
-    DatastoreServiceOptions copy = original.toBuilder().build();
+    DatastoreOptions original = options.namespace("ns1").force(true).build();
+    DatastoreOptions copy = original.toBuilder().build();
     assertEquals(original.projectId(), copy.projectId());
     assertEquals(original.namespace(), copy.namespace());
     assertEquals(original.host(), copy.host());
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
deleted file mode 100644
index 1a06de833cf2..000000000000
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceExceptionTest.java
+++ /dev/null
@@ -1 +0,0 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

import com.google.gcloud.datastore.DatastoreServiceException.Code;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.junit.Test;

public class DatastoreServiceExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retryable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreServiceException exception = new DatastoreServiceException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreServiceException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreServiceException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreServiceException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreServiceException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java
similarity index 95%
rename from gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
rename to gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java
index 535fcb5f13e1..156f9684f8ba 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreServiceTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreTest.java
@@ -51,7 +51,7 @@
 import java.util.List;
 
 @RunWith(JUnit4.class)
-public class DatastoreServiceTest {
+public class DatastoreTest {
 
   private static final String PROJECT_ID = LocalGcdHelper.DEFAULT_PROJECT_ID;
   private static final String KIND1 = "kind1";
@@ -97,8 +97,8 @@ public class DatastoreServiceTest {
   private static final Entity ENTITY3 = Entity.builder(ENTITY1).key(KEY3).remove("str")
       .set("null", NULL_VALUE).set("partial1", PARTIAL_ENTITY2).set("partial2", ENTITY2).build();
 
-  private DatastoreServiceOptions options;
-  private DatastoreService datastore;
+  private DatastoreOptions options;
+  private Datastore datastore;
 
   private static LocalGcdHelper gcdHelper;
 
@@ -111,11 +111,11 @@ public static void beforeClass() throws IOException, InterruptedException {
 
   @Before
   public void setUp() throws IOException, InterruptedException {
-    options = DatastoreServiceOptions.builder()
+    options = DatastoreOptions.builder()
         .projectId(PROJECT_ID)
         .host("http://localhost:" + LocalGcdHelper.PORT)
         .build();
-    datastore = DatastoreServiceFactory.instance().get(options);
+    datastore = DatastoreFactory.instance().get(options);
     StructuredQuery query = Query.keyQueryBuilder().build();
     QueryResults result = datastore.run(query);
     datastore.delete(Iterators.toArray(result, Key.class));
@@ -155,14 +155,14 @@ public void testNewTransactionCommit() {
     try {
       transaction.commit();
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
 
     try {
       transaction.rollback();
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
 
@@ -185,8 +185,8 @@ public void testTransactionWithRead() {
     try {
       transaction.commit();
       fail("Expecting a failure");
-    } catch (DatastoreServiceException expected) {
-      assertEquals(DatastoreServiceException.Code.ABORTED, expected.code());
+    } catch (DatastoreException expected) {
+      assertEquals(DatastoreException.Code.ABORTED, expected.code());
     }
   }
 
@@ -213,8 +213,8 @@ public void testTransactionWithQuery() {
     try {
       transaction.commit();
       fail("Expecting a failure");
-    } catch (DatastoreServiceException expected) {
-      assertEquals(DatastoreServiceException.Code.ABORTED, expected.code());
+    } catch (DatastoreException expected) {
+      assertEquals(DatastoreException.Code.ABORTED, expected.code());
     }
   }
 
@@ -232,7 +232,7 @@ public void testNewTransactionRollback() {
     try {
       transaction.commit();
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
 
@@ -249,28 +249,28 @@ private void verifyNotUsable(DatastoreWriter writer) {
     try {
       writer.add(ENTITY3);
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
 
     try {
       writer.put(ENTITY3);
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
 
     try {
       writer.update(ENTITY3);
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
 
     try {
       writer.delete(ENTITY3.key());
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
   }
@@ -315,7 +315,7 @@ public void testNewBatch() {
     try {
       batch.submit();
       fail("Expecting a failure");
-    } catch (DatastoreServiceException ex) {
+    } catch (DatastoreException ex) {
       // expected to fail
     }
     verifyNotUsable(batch);
@@ -535,7 +535,7 @@ public void testGetArray() {
     try {
       entity3.getString("str");
       fail("Expecting a failure");
-    } catch (DatastoreServiceException expected) {
+    } catch (DatastoreException expected) {
       // expected - no such property
     }
     assertFalse(result.hasNext());
@@ -552,7 +552,7 @@ public void testAddEntity() {
     try {
       datastore.add(ENTITY1);
       fail("Expecting a failure");
-    } catch (DatastoreServiceException expected) {
+    } catch (DatastoreException expected) {
       // expected;
     }
 
@@ -578,7 +578,7 @@ public void testUpdate() {
     try {
       datastore.update(ENTITY3);
       fail("Expecting a failure");
-    } catch (DatastoreServiceException expected) {
+    } catch (DatastoreException expected) {
       // expected;
     }
     datastore.add(ENTITY3);
@@ -642,17 +642,17 @@ public void testRetires() throws Exception {
         .addFound(EntityResult.newBuilder().setEntity(ENTITY1.toPb())).build();
     DatastoreRpcFactory rpcFactoryMock = EasyMock.createStrictMock(DatastoreRpcFactory.class);
     DatastoreRpc rpcMock = EasyMock.createStrictMock(DatastoreRpc.class);
-    EasyMock.expect(rpcFactoryMock.create(EasyMock.anyObject(DatastoreServiceOptions.class)))
+    EasyMock.expect(rpcFactoryMock.create(EasyMock.anyObject(DatastoreOptions.class)))
         .andReturn(rpcMock);
     EasyMock.expect(rpcMock.lookup(requestPb))
         .andThrow(new DatastoreRpc.DatastoreRpcException(Reason.UNAVAILABLE))
         .andReturn(responsePb);
     EasyMock.replay(rpcFactoryMock, rpcMock);
-    DatastoreServiceOptions options = this.options.toBuilder()
+    DatastoreOptions options = this.options.toBuilder()
         .retryParams(RetryParams.getDefaultInstance())
         .serviceRpcFactory(rpcFactoryMock)
         .build();
-    DatastoreService datastore = DatastoreServiceFactory.instance().get(options);
+    Datastore datastore = DatastoreFactory.instance().get(options);
     Entity entity = datastore.get(KEY1);
     assertEquals(ENTITY1, entity);
     EasyMock.verify(rpcFactoryMock, rpcMock);
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ListValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ListValueTest.java
index 04fdbec54727..36e3571d49ac 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ListValueTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ListValueTest.java
@@ -45,7 +45,7 @@ public void testOf() throws Exception {
     assertFalse(value.hasMeaning());
   }
 
-  @Test(expected = DatastoreServiceException.class)
+  @Test(expected = DatastoreException.class)
   public void testIndexedCannotBeSpecified() {
     ListValue.builder().indexed(false);
   }
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 1e827971c7fe..9574f1e246d2 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -132,12 +132,12 @@ public class SerializationTest {
 
   @Test
   public void testServiceOptions() throws Exception {
-    DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+    DatastoreOptions options = DatastoreOptions.builder()
         .authCredentials(AuthCredentials.createForAppEngine())
         .normalizeDataset(false)
         .projectId("ds1")
         .build();
-    DatastoreServiceOptions serializedCopy = serializeAndDeserialize(options);
+    DatastoreOptions serializedCopy = serializeAndDeserialize(options);
     assertEquals(options, serializedCopy);
 
     options = options.toBuilder()
diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index 3d7feae66e8c..9188117e4327 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -16,9 +16,9 @@
 
 package com.google.gcloud.examples;
 
-import com.google.gcloud.datastore.DatastoreService;
-import com.google.gcloud.datastore.DatastoreServiceFactory;
-import com.google.gcloud.datastore.DatastoreServiceOptions;
+import com.google.gcloud.datastore.Datastore;
+import com.google.gcloud.datastore.DatastoreFactory;
+import com.google.gcloud.datastore.DatastoreOptions;
 import com.google.gcloud.datastore.DateTime;
 import com.google.gcloud.datastore.Entity;
 import com.google.gcloud.datastore.FullEntity;
@@ -172,21 +172,21 @@ public String getRequiredParams() {
 
   public static void main(String... args) {
     DatastoreAction action = null;
-    DatastoreService datastore = null;
+    Datastore datastore = null;
     Key key = null;
     String projectId = args.length > 0 ? args[0] : null;
     // If you want to access a local Datastore running via the gcd sdk, do
-    //   DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+    //   DatastoreOptions options = DatastoreOptions.builder()
     //       .projectId(projectId)
     //       .namespace(NAMESPACE)
     //       .host("http://localhost:8080")
     //       .build();
-    DatastoreServiceOptions options = DatastoreServiceOptions.builder()
+    DatastoreOptions options = DatastoreOptions.builder()
         .projectId(projectId)
         .namespace(NAMESPACE)
         .build();
     String name = args.length > 1 ? args[1] : System.getProperty("user.name");
-    datastore = DatastoreServiceFactory.instance().get(options);
+    datastore = DatastoreFactory.instance().get(options);
     KeyFactory keyFactory = datastore.newKeyFactory().kind(USER_KIND);
     key = keyFactory.newKey(name);
     String actionName = args.length > 2 ? args[2].toLowerCase() : DEFAULT_ACTION;
diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
index b0d44c292d2c..b2e5efca4a55 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -24,11 +24,11 @@
 import com.google.gcloud.storage.BlobReadChannel;
 import com.google.gcloud.storage.BlobWriteChannel;
 import com.google.gcloud.storage.Bucket;
-import com.google.gcloud.storage.StorageService;
-import com.google.gcloud.storage.StorageService.ComposeRequest;
-import com.google.gcloud.storage.StorageService.CopyRequest;
-import com.google.gcloud.storage.StorageServiceFactory;
-import com.google.gcloud.storage.StorageServiceOptions;
+import com.google.gcloud.storage.Storage;
+import com.google.gcloud.storage.Storage.ComposeRequest;
+import com.google.gcloud.storage.Storage.CopyRequest;
+import com.google.gcloud.storage.StorageFactory;
+import com.google.gcloud.storage.StorageOptions;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -73,7 +73,7 @@ public class StorageExample {
 
   private static abstract class StorageAction {
 
-    abstract void run(StorageService storage, T request) throws Exception;
+    abstract void run(Storage storage, T request) throws Exception;
 
     abstract T parse(String... args) throws IllegalArgumentException, IOException;
 
@@ -127,7 +127,7 @@ public String params() {
    */
   private static class InfoAction extends BlobsAction {
     @Override
-    public void run(StorageService storage, Blob... blobs) {
+    public void run(Storage storage, Blob... blobs) {
       if (blobs.length == 1) {
         if (blobs[0].name().isEmpty()) {
           // get Bucket
@@ -174,7 +174,7 @@ public String params() {
    */
   private static class DeleteAction extends BlobsAction {
     @Override
-    public void run(StorageService storage, Blob... blobs) {
+    public void run(Storage storage, Blob... blobs) {
       if (blobs.length == 1) {
         boolean wasDeleted = storage.delete(blobs[0].bucket(), blobs[0].name());
         if (wasDeleted) {
@@ -218,7 +218,7 @@ String parse(String... args) {
     }
 
     @Override
-    public void run(StorageService storage, String bucket) {
+    public void run(Storage storage, String bucket) {
       if (bucket == null) {
         // list buckets
         for (Bucket b : storage.list()) {
@@ -245,11 +245,11 @@ public String params() {
    */
   private static class UploadAction extends StorageAction> {
     @Override
-    public void run(StorageService storage, Tuple tuple) throws Exception {
+    public void run(Storage storage, Tuple tuple) throws Exception {
       run(storage, tuple.x(), tuple.y());
     }
 
-    private void run(StorageService storage, Path uploadFrom, Blob blob) throws IOException {
+    private void run(Storage storage, Path uploadFrom, Blob blob) throws IOException {
       if (Files.size(uploadFrom) > 1_000_000) {
         // When content is not available or large (1MB or more) it is recommended
         // to write it in chunks via the blob's channel writer.
@@ -301,11 +301,11 @@ public String params() {
   private static class DownloadAction extends StorageAction> {
 
     @Override
-    public void run(StorageService storage, Tuple tuple) throws IOException {
+    public void run(Storage storage, Tuple tuple) throws IOException {
       run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y());
     }
 
-    private void run(StorageService storage, String bucket, String blobName, Path downloadTo)
+    private void run(Storage storage, String bucket, String blobName, Path downloadTo)
         throws IOException {
       Blob blob = storage.get(bucket, blobName);
       if (blob == null) {
@@ -369,7 +369,7 @@ public String params() {
    */
   private static class CopyAction extends StorageAction {
     @Override
-    public void run(StorageService storage, CopyRequest request) {
+    public void run(Storage storage, CopyRequest request) {
       Blob copiedBlob = storage.copy(request);
       System.out.println("Copied " + copiedBlob);
     }
@@ -395,7 +395,7 @@ public String params() {
    */
   private static class ComposeAction extends StorageAction {
     @Override
-    public void run(StorageService storage, ComposeRequest request) {
+    public void run(Storage storage, ComposeRequest request) {
       Blob composedBlob = storage.compose(request);
       System.out.println("Composed " + composedBlob);
     }
@@ -427,12 +427,12 @@ public String params() {
   private static class UpdateMetadata extends StorageAction>> {
 
     @Override
-    public void run(StorageService storage, Tuple> tuple)
+    public void run(Storage storage, Tuple> tuple)
         throws IOException {
       run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y());
     }
 
-    private void run(StorageService storage, String bucket, String blobName,
+    private void run(Storage storage, String bucket, String blobName,
         Map metadata) {
       Blob blob = storage.get(bucket, blobName);
       if (blob == null) {
@@ -499,8 +499,8 @@ public static void main(String... args) throws Exception {
       printUsage();
       return;
     }
-    StorageServiceOptions.Builder optionsBuilder =
-        StorageServiceOptions.builder().retryParams(RetryParams.getDefaultInstance());
+    StorageOptions.Builder optionsBuilder =
+        StorageOptions.builder().retryParams(RetryParams.getDefaultInstance());
     StorageAction action;
     if (args.length >= 2 && !ACTIONS.containsKey(args[0])) {
       optionsBuilder.projectId(args[0]);
@@ -515,7 +515,7 @@ public static void main(String... args) throws Exception {
       printUsage();
       return;
     }
-    StorageService storage = StorageServiceFactory.instance().get(optionsBuilder.build());
+    Storage storage = StorageFactory.instance().get(optionsBuilder.build());
     Object request;
     try {
       request = action.parse(args);
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
index e27d837d7173..e88fbb473934 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java
@@ -58,8 +58,8 @@
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
-import com.google.gcloud.storage.StorageServiceException;
-import com.google.gcloud.storage.StorageServiceOptions;
+import com.google.gcloud.storage.StorageException;
+import com.google.gcloud.storage.StorageOptions;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -71,13 +71,13 @@
 public class DefaultStorageRpc implements StorageRpc {
 
   public static final String DEFAULT_PROJECTION = "full";
-  private final StorageServiceOptions options;
+  private final StorageOptions options;
   private final Storage storage;
 
   // see: https://cloud.google.com/storage/docs/concepts-techniques#practices
   private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 408);
 
-  public DefaultStorageRpc(StorageServiceOptions options) {
+  public DefaultStorageRpc(StorageOptions options) {
     HttpTransport transport = options.httpTransportFactory().create();
     HttpRequestInitializer initializer = options.httpRequestInitializer();
     this.options = options;
@@ -87,25 +87,25 @@ public DefaultStorageRpc(StorageServiceOptions options) {
     // Todo: make sure nulls are being used as Data.asNull()
   }
 
-  private static StorageServiceException translate(IOException exception) {
-    StorageServiceException translated;
+  private static StorageException translate(IOException exception) {
+    StorageException translated;
     if (exception instanceof GoogleJsonResponseException) {
       translated = translate(((GoogleJsonResponseException) exception).getDetails());
     } else {
-      translated = new StorageServiceException(0, exception.getMessage(), false);
+      translated = new StorageException(0, exception.getMessage(), false);
     }
     translated.initCause(exception);
     return translated;
   }
 
-  private static StorageServiceException translate(GoogleJsonError exception) {
+  private static StorageException translate(GoogleJsonError exception) {
     boolean retryable = RETRYABLE_CODES.contains(exception.getCode())
         || "InternalError".equals(exception.getMessage());
-    return new StorageServiceException(exception.getCode(), exception.getMessage(), retryable);
+    return new StorageException(exception.getCode(), exception.getMessage(), retryable);
   }
 
   @Override
-  public Bucket create(Bucket bucket, Map options) throws StorageServiceException {
+  public Bucket create(Bucket bucket, Map options) throws StorageException {
     try {
       return storage.buckets()
           .insert(this.options.projectId(), bucket)
@@ -120,7 +120,7 @@ public Bucket create(Bucket bucket, Map options) throws StorageServic
 
   @Override
   public StorageObject create(StorageObject storageObject, final byte[] content,
-      Map options) throws StorageServiceException {
+      Map options) throws StorageException {
     try {
       return storage.objects()
           .insert(storageObject.getBucket(), storageObject,
@@ -253,7 +253,7 @@ public boolean delete(Bucket bucket, Map options) {
           .execute();
       return true;
     } catch (IOException ex) {
-      StorageServiceException serviceException = translate(ex);
+      StorageException serviceException = translate(ex);
       if (serviceException.code() == 404) {
         return false;
       }
@@ -267,7 +267,7 @@ public boolean delete(StorageObject blob, Map options) {
       deleteRequest(blob, options).execute();
       return true;
     } catch (IOException ex) {
-      StorageServiceException serviceException = translate(ex);
+      StorageException serviceException = translate(ex);
       if (serviceException.code() == 404) {
         return false;
       }
@@ -288,7 +288,7 @@ private Storage.Objects.Delete deleteRequest(StorageObject blob, Map
 
   @Override
   public StorageObject compose(Iterable sources, StorageObject target,
-      Map targetOptions) throws StorageServiceException {
+      Map targetOptions) throws StorageException {
     ComposeRequest request = new ComposeRequest();
     if (target.getContentType() == null) {
       // todo: remove once this is no longer requirement (b/20681287).
@@ -322,7 +322,7 @@ public StorageObject compose(Iterable sources, StorageObject targ
 
   @Override
   public StorageObject copy(StorageObject source, Map sourceOptions,
-      StorageObject target, Map targetOptions) throws StorageServiceException {
+      StorageObject target, Map targetOptions) throws StorageException {
     try {
       return storage
           .objects()
@@ -345,7 +345,7 @@ public StorageObject copy(StorageObject source, Map sourceOptions,
 
   @Override
   public byte[] load(StorageObject from, Map options)
-      throws StorageServiceException {
+      throws StorageException {
     try {
       Storage.Objects.Get getRequest = storage.objects()
           .get(from.getBucket(), from.getName())
@@ -363,25 +363,25 @@ public byte[] load(StorageObject from, Map options)
   }
 
   @Override
-  public BatchResponse batch(BatchRequest request) throws StorageServiceException {
+  public BatchResponse batch(BatchRequest request) throws StorageException {
     com.google.api.client.googleapis.batch.BatchRequest batch = storage.batch();
-    final Map> deletes =
+    final Map> deletes =
         Maps.newConcurrentMap();
-    final Map> updates =
+    final Map> updates =
         Maps.newConcurrentMap();
-    final Map> gets =
+    final Map> gets =
         Maps.newConcurrentMap();
     try {
       for (final Tuple> tuple : request.toDelete) {
         deleteRequest(tuple.x(), tuple.y()).queue(batch, new JsonBatchCallback() {
           @Override
           public void onSuccess(Void ignore, HttpHeaders responseHeaders) {
-            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
+            deletes.put(tuple.x(), Tuple.of(Boolean.TRUE, null));
           }
 
           @Override
           public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
-            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
+            deletes.put(tuple.x(), Tuple.of(null, translate(e)));
           }
         });
       }
@@ -390,13 +390,13 @@ public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
           @Override
           public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
             updates.put(tuple.x(),
-                Tuple.of(storageObject, null));
+                Tuple.of(storageObject, null));
           }
 
           @Override
           public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
             updates.put(tuple.x(),
-                Tuple.of(null, translate(e)));
+                Tuple.of(null, translate(e)));
           }
         });
       }
@@ -405,13 +405,13 @@ public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
           @Override
           public void onSuccess(StorageObject storageObject, HttpHeaders responseHeaders) {
             gets.put(tuple.x(),
-                Tuple.of(storageObject, null));
+                Tuple.of(storageObject, null));
           }
 
           @Override
           public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
             gets.put(tuple.x(),
-                Tuple.of(null, translate(e)));
+                Tuple.of(null, translate(e)));
           }
         });
       }
@@ -424,7 +424,7 @@ public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) {
 
   @Override
   public byte[] read(StorageObject from, Map options, long position, int bytes)
-      throws StorageServiceException {
+      throws StorageException {
     try {
       Get req = storage.objects().get(from.getBucket(), from.getName());
       req.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(options))
@@ -445,7 +445,7 @@ public byte[] read(StorageObject from, Map options, long position, in
 
   @Override
   public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
-      long destOffset, int length, boolean last) throws StorageServiceException {
+      long destOffset, int length, boolean last) throws StorageException {
     try {
       GenericUrl url = new GenericUrl(uploadId);
       HttpRequest httpRequest = storage.getRequestFactory().buildPostRequest(url,
@@ -487,7 +487,7 @@ public void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObj
 
   @Override
   public String open(StorageObject object, Map options)
-      throws StorageServiceException {
+      throws StorageException {
     try {
       Insert req = storage.objects().insert(object.getBucket(), object);
       GenericUrl url = req.buildHttpRequest().getUrl();
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
index ab1e9affbbce..5d14a5e786c7 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageServiceException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageServiceException;

  Tuple> list(Map options) throws StorageServiceException;

  Tuple> list(String bucket, Map options)
      throws StorageServiceException;

  Bucket get(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject get(StorageObject object, Map options)
      throws StorageServiceException;

  Bucket patch(Bucket bucket, Map options) throws StorageServiceException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageServiceException;

  boolean delete(Bucket bucket, Map options) throws StorageServiceException;

  boolean delete(StorageObject object, Map options) throws StorageServiceException;

  BatchResponse batch(BatchRequest request) throws StorageServiceException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageServiceException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageServiceException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageServiceException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageServiceException;

  String open(StorageObject object, Map options) throws StorageServiceException;

  void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageServiceException;
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageException;

  Tuple> list(Map options) throws StorageException;

  Tuple> list(String bucket, Map options)
      throws StorageException;

  Bucket get(Bucket bucket, Map options) throws StorageException;

  StorageObject get(StorageObject object, Map options)
      throws StorageException;

  Bucket patch(Bucket bucket, Map options) throws StorageException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageException;

  boolean delete(Bucket bucket, Map options) throws StorageException;

  boolean delete(StorageObject object, Map options) throws StorageException;

  BatchResponse batch(BatchRequest request) throws StorageException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageException;

  String open(StorageObject object, Map options) throws StorageException;

  void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageException;
}
\ No newline at end of file
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
index f693e63018ce..f4959d617d17 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpcFactory.java
@@ -16,12 +16,12 @@
 
 package com.google.gcloud.spi;
 
-import com.google.gcloud.storage.StorageServiceOptions;
+import com.google.gcloud.storage.StorageOptions;
 
 /**
  * An interface for Storage RPC factory.
  * Implementation will be loaded via {@link java.util.ServiceLoader}.
  */
-public interface StorageRpcFactory extends ServiceRpcFactory {
+public interface StorageRpcFactory extends ServiceRpcFactory {
 }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 4d2f0cab8c96..5cea321b0071 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -18,8 +18,8 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
-import com.google.gcloud.storage.StorageService.BlobSourceOption;
-import com.google.gcloud.storage.StorageService.BlobTargetOption;
+import com.google.gcloud.storage.Storage.BlobSourceOption;
+import com.google.gcloud.storage.Storage.BlobTargetOption;
 
 import java.io.Serializable;
 import java.util.LinkedHashMap;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
index 7be1b22f6cc7..1a05fba819c4 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -40,7 +40,7 @@ public static class Result implements Serializable {
     private static final Result EMPTY = new BatchResponse.Result(null);
 
     private final T value;
-    private final StorageServiceException exception;
+    private final StorageException exception;
 
 
     public Result(T value) {
@@ -48,7 +48,7 @@ public Result(T value) {
       this.exception = null;
     }
 
-    public Result(StorageServiceException exception) {
+    public Result(StorageException exception) {
       this.exception = exception;
       this.value = null;
     }
@@ -60,9 +60,9 @@ static  Result of(T value) {
     /**
      * Returns the result.
      *
-     * @throws StorageServiceException if failed
+     * @throws StorageException if failed
      */
-    public T get() throws StorageServiceException {
+    public T get() throws StorageException {
       if (failed()) {
         throw failure();
       }
@@ -72,7 +72,7 @@ public T get() throws StorageServiceException {
     /**
      * Returns the failure or {@code null} if was successful.
      */
-    public StorageServiceException failure() {
+    public StorageException failure() {
       return exception;
     }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
index 27d37b127d55..5a5d165751c4 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
@@ -36,7 +36,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
   private static final long serialVersionUID = 4821762590742862669L;
 
-  private final StorageServiceOptions serviceOptions;
+  private final StorageOptions serviceOptions;
   private final Blob blob;
   private final Map requestOptions;
   private int position;
@@ -49,7 +49,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private transient int bufferPos;
   private transient byte[] buffer;
 
-  BlobReadChannelImpl(StorageServiceOptions serviceOptions, Blob blob,
+  BlobReadChannelImpl(StorageOptions serviceOptions, Blob blob,
       Map requestOptions) {
     this.serviceOptions = serviceOptions;
     this.blob = blob;
@@ -124,7 +124,7 @@ public int read(ByteBuffer byteBuffer) throws IOException {
         public byte[] call() {
           return storageRpc.read(storageObject, requestOptions, position, toRead);
         }
-      }, serviceOptions.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
+      }, serviceOptions.retryParams(), StorageImpl.EXCEPTION_HANDLER);
       if (toRead > buffer.length) {
         endOfStream = true;
         if (buffer.length == 0) {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
index 2b8e66cc33ce..5e41ed00fc54 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
@@ -38,7 +38,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private static final int MIN_CHUNK_SIZE = 256 * 1024;
   private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE;
 
-  private final StorageServiceOptions options;
+  private final StorageOptions options;
   private final Blob blob;
   private final String uploadId;
   private int position;
@@ -50,7 +50,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
 
-  public BlobWriterChannelImpl(StorageServiceOptions options, Blob blob,
+  public BlobWriterChannelImpl(StorageOptions options, Blob blob,
       Map optionsMap) {
     this.options = options;
     this.blob = blob;
@@ -73,7 +73,7 @@ private void flush(boolean compact) {
         public void run() {
           storageRpc.write(uploadId, buffer, 0, storageObject, position, length, false);
         }
-      }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
+      }), options.retryParams(), StorageImpl.EXCEPTION_HANDLER);
       position += length;
       limit -= length;
       byte[] temp = new byte[compact ? limit : chunkSize];
@@ -129,7 +129,7 @@ public void close() throws IOException {
         public void run() {
           storageRpc.write(uploadId, buffer, 0, storageObject, position, limit, true);
         }
-      }), options.retryParams(), StorageServiceImpl.EXCEPTION_HANDLER);
+      }), options.retryParams(), StorageImpl.EXCEPTION_HANDLER);
       position += buffer.length;
       isOpen = false;
       buffer = null;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
similarity index 94%
rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index 6fefb3af3b16..d68d30ade556 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageService.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -36,7 +36,7 @@
  *
  * @see Google Cloud Storage
  */
-public interface StorageService extends Service {
+public interface Storage extends Service {
 
   enum PredefinedAcl {
     AUTHENTICATED_READ("authenticatedRead"),
@@ -411,7 +411,7 @@ public static Builder builder() {
    * Create a new bucket.
    *
    * @return a complete bucket information.
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Bucket create(Bucket bucket, BucketTargetOption... options);
 
@@ -419,35 +419,35 @@ public static Builder builder() {
    * Create a new blob.
    *
    * @return a complete blob information.
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Blob create(Blob blob, byte[] content, BlobTargetOption... options);
 
   /**
    * Return the requested bucket or {@code null} if not found.
    *
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Bucket get(String bucket, BucketSourceOption... options);
 
   /**
    * Return the requested blob or {@code null} if not found.
    *
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Blob get(String bucket, String blob, BlobSourceOption... options);
 
   /**
    * List the project's buckets.
    *
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   ListResult list(BucketListOption... options);
 
   /**
    * List the bucket's blobs.
    *
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   ListResult list(String bucket, BlobListOption... options);
 
@@ -455,7 +455,7 @@ public static Builder builder() {
    * Update bucket information.
    *
    * @return the updated bucket
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Bucket update(Bucket bucket, BucketTargetOption... options);
 
@@ -463,7 +463,7 @@ public static Builder builder() {
    * Update blob information.
    *
    * @return the updated blob
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Blob update(Blob blob, BlobTargetOption... options);
 
@@ -471,7 +471,7 @@ public static Builder builder() {
    * Delete the requested bucket.
    *
    * @return true if bucket was deleted
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   boolean delete(String bucket, BucketSourceOption... options);
 
@@ -479,7 +479,7 @@ public static Builder builder() {
    * Delete the requested blob.
    *
    * @return true if blob was deleted
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   boolean delete(String bucket, String blob, BlobSourceOption... options);
 
@@ -487,7 +487,7 @@ public static Builder builder() {
    * Send a compose request.
    *
    * @return the composed blob.
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Blob compose(ComposeRequest composeRequest);
 
@@ -495,7 +495,7 @@ public static Builder builder() {
    * Send a copy request.
    *
    * @return the copied blob.
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   Blob copy(CopyRequest copyRequest);
 
@@ -503,7 +503,7 @@ public static Builder builder() {
    * Load the content of the given blob.
    *
    * @return the blob's content.
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   byte[] load(String bucket, String blob, BlobSourceOption... options);
 
@@ -511,21 +511,21 @@ public static Builder builder() {
    * Send a batch request.
    *
    * @return the batch response
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   BatchResponse apply(BatchRequest batchRequest);
 
   /**
    * Return a channel for reading the blob's content.
    *
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   BlobReadChannel reader(String bucket, String blob, BlobSourceOption... options);
 
   /**
    * Create a blob and return a channel for writing its content.
    *
-   * @throws StorageServiceException upon failure
+   * @throws StorageException upon failure
    */
   BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceException.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageException.java
similarity index 61%
rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceException.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageException.java
index 900a762d46d2..9bf073bef6b4 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceException.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageException.java
@@ -1 +1 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

/**
 * Storage service exception.
 *
 * @see Google Cloud
 *      Storage error codes
 */
public class StorageServiceException extends RuntimeException {

  private static final long serialVersionUID = -3748432005065428084L;

  private final int code;
  private final boolean retryable;

  public StorageServiceException(int code, String message, boolean retryable) {
    super(message);
    this.code = code;
    this.retryable = retryable;
  }

  /**
   * Returns the code associated with this exception.
   */
  public int code() {
    return code;
  }

  public boolean retryable() {
    return retryable;
  }
}
\ No newline at end of file
+/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

/**
 * Storage service exception.
 *
 * @see Google Cloud
 *      Storage error codes
 */
public class StorageException extends RuntimeException {

  private static final long serialVersionUID = -3748432005065428084L;

  private final int code;
  private final boolean retryable;

  public StorageException(int code, String message, boolean retryable) {
    super(message);
    this.code = code;
    this.retryable = retryable;
  }

  /**
   * Returns the code associated with this exception.
   */
  public int code() {
    return code;
  }

  public boolean retryable() {
    return retryable;
  }
}
\ No newline at end of file
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageFactory.java
similarity index 62%
rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageFactory.java
index 51c7c1812e6f..e269f0c9d92b 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceFactory.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageFactory.java
@@ -18,26 +18,26 @@
 
 
 /**
- * A base class for StorageService factories.
+ * A base class for Storage factories.
  */
-public abstract class StorageServiceFactory {
+public abstract class StorageFactory {
 
-  private static final StorageServiceFactory INSTANCE = new StorageServiceFactory() {
+  private static final StorageFactory INSTANCE = new StorageFactory() {
     @Override
-    public StorageService get(StorageServiceOptions options) {
-      return new StorageServiceImpl(options);
+    public Storage get(StorageOptions options) {
+      return new StorageImpl(options);
     }
   };
 
   /**
    * Returns the default factory instance.
    */
-  public static StorageServiceFactory instance() {
+  public static StorageFactory instance() {
     return INSTANCE;
   }
 
   /**
-   * Returns a {@code StorageService} for the given options.
+   * Returns a {@code Storage} service for the given options.
    */
-  public abstract StorageService get(StorageServiceOptions options);
+  public abstract Storage get(StorageOptions options);
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
similarity index 94%
rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index 95b95141be14..940022eddcbd 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -53,7 +53,7 @@
 import java.util.Set;
 import java.util.concurrent.Callable;
 
-final class StorageServiceImpl extends BaseService implements StorageService {
+final class StorageImpl extends BaseService implements Storage {
 
   private static final Interceptor EXCEPTION_HANDLER_INTERCEPTOR = new Interceptor() {
 
@@ -66,8 +66,8 @@ public RetryResult afterEval(Exception exception, RetryResult retryResult) {
 
     @Override
     public RetryResult beforeEval(Exception exception) {
-      if (exception instanceof StorageServiceException) {
-        boolean retriable = ((StorageServiceException) exception).retryable();
+      if (exception instanceof StorageException) {
+        boolean retriable = ((StorageException) exception).retryable();
         return retriable ? Interceptor.RetryResult.RETRY : Interceptor.RetryResult.ABORT;
       }
       return null;
@@ -79,7 +79,7 @@ public RetryResult beforeEval(Exception exception) {
 
   private final StorageRpc storageRpc;
 
-  StorageServiceImpl(StorageServiceOptions options) {
+  StorageImpl(StorageOptions options) {
     super(options);
     storageRpc = options.storageRpc();
     // todo: replace nulls with Value.asNull (per toPb)
@@ -124,7 +124,7 @@ public Bucket get(String bucket, BucketSourceOption... options) {
           public com.google.api.services.storage.model.Bucket call() {
             try {
               return storageRpc.get(bucketPb, optionsMap);
-            } catch (StorageServiceException ex) {
+            } catch (StorageException ex) {
               if (ex.code() == HTTP_NOT_FOUND) {
                 return null;
               }
@@ -144,7 +144,7 @@ public Blob get(String bucket, String blob, BlobSourceOption... options) {
       public StorageObject call() {
         try {
           return storageRpc.get(storedObject, optionsMap);
-        } catch (StorageServiceException ex) {
+        } catch (StorageException ex) {
           if (ex.code() == HTTP_NOT_FOUND) {
             return null;
           }
@@ -160,9 +160,9 @@ private static abstract class BasePageFetcher
 
     private static final long serialVersionUID = 8236329004030295223L;
     protected final Map requestOptions;
-    protected final StorageServiceOptions serviceOptions;
+    protected final StorageOptions serviceOptions;
 
-    BasePageFetcher(StorageServiceOptions serviceOptions, String cursor,
+    BasePageFetcher(StorageOptions serviceOptions, String cursor,
         Map optionMap) {
       this.serviceOptions = serviceOptions;
       ImmutableMap.Builder builder = ImmutableMap.builder();
@@ -180,7 +180,7 @@ private static class BucketPageFetcher extends BasePageFetcher {
 
     private static final long serialVersionUID = -5490616010200159174L;
 
-    BucketPageFetcher(StorageServiceOptions serviceOptions, String cursor,
+    BucketPageFetcher(StorageOptions serviceOptions, String cursor,
         Map optionMap) {
       super(serviceOptions, cursor, optionMap);
     }
@@ -196,7 +196,7 @@ private static class BlobPageFetcher extends BasePageFetcher {
     private static final long serialVersionUID = -5490616010200159174L;
     private final String bucket;
 
-    BlobPageFetcher(String bucket, StorageServiceOptions serviceOptions, String cursor,
+    BlobPageFetcher(String bucket, StorageOptions serviceOptions, String cursor,
         Map optionMap) {
       super(serviceOptions, cursor, optionMap);
       this.bucket = bucket;
@@ -213,7 +213,7 @@ public ListResult list(BucketListOption... options) {
     return listBuckets(options(), optionMap(options));
   }
 
-  private static ListResult listBuckets(final StorageServiceOptions serviceOptions,
+  private static ListResult listBuckets(final StorageOptions serviceOptions,
       final Map optionsMap) {
     Tuple> result = runWithRetries(
         new Callable>>() {
@@ -239,7 +239,7 @@ public ListResult list(final String bucket, BlobListOption... options) {
   }
 
   private static ListResult listBlobs(final String bucket,
-      final StorageServiceOptions serviceOptions, final Map optionsMap) {
+      final StorageOptions serviceOptions, final Map optionsMap) {
     Tuple> result = runWithRetries(
         new Callable>>() {
           @Override
@@ -396,16 +396,16 @@ public BatchResponse apply(BatchRequest batchRequest) {
 
   private  List> transformBatchResult(
       Iterable>> request,
-      Map> results, Function transform,
+      Map> results, Function transform,
       int... nullOnErrorCodes) {
     Set nullOnErrorCodesSet = Sets.newHashSet(Ints.asList(nullOnErrorCodes));
     List> response = Lists.newArrayListWithCapacity(results.size());
     for (Tuple tuple : request) {
-      Tuple result = results.get(tuple.x());
+      Tuple result = results.get(tuple.x());
       if (result.x() != null) {
         response.add(BatchResponse.Result.of(transform.apply(result.x())));
       } else {
-        StorageServiceException exception = result.y();
+        StorageException exception = result.y();
         if (nullOnErrorCodesSet.contains(exception.code())) {
           //noinspection unchecked
           response.add(BatchResponse.Result.empty());
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
similarity index 83%
rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
index 1bb10743de68..9e4ba2b72407 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageServiceOptions.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
@@ -26,7 +26,7 @@
 import java.util.Objects;
 import java.util.Set;
 
-public class StorageServiceOptions extends ServiceOptions {
+public class StorageOptions extends ServiceOptions {
 
   private static final long serialVersionUID = -7804860602287801084L;
   private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control";
@@ -37,13 +37,13 @@ public class StorageServiceOptions extends ServiceOptions {
+      ServiceOptions.Builder {
 
     private String pathDelimiter;
 
     private Builder() {}
 
-    private Builder(StorageServiceOptions options) {
+    private Builder(StorageOptions options) {
       super(options);
     }
 
@@ -53,12 +53,12 @@ public Builder pathDelimiter(String pathDelimiter) {
     }
 
     @Override
-    public StorageServiceOptions build() {
-      return new StorageServiceOptions(this);
+    public StorageOptions build() {
+      return new StorageOptions(this);
     }
   }
 
-  private StorageServiceOptions(Builder builder) {
+  private StorageOptions(Builder builder) {
     super(builder);
     pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
     // todo: consider providing read-timeout
@@ -100,14 +100,14 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (!(obj instanceof StorageServiceOptions)) {
+    if (!(obj instanceof StorageOptions)) {
       return false;
     }
-    StorageServiceOptions other = (StorageServiceOptions) obj;
+    StorageOptions other = (StorageOptions) obj;
     return isEquals(other) && Objects.equals(pathDelimiter, other.pathDelimiter);
   }
 
-  public static StorageServiceOptions defaultInstance() {
+  public static StorageOptions defaultInstance() {
     return builder().build();
   }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
index c02b743d0977..c648ea88e39d 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
@@ -19,8 +19,8 @@
  *
  * 
    A simple usage example:
  * 
    {@code
- * StorageServiceOptions options = StorageServiceOptions.builder().projectId("project").build();
- * StorageService storage = StorageServiceFactory.instance().get(options);
+ * StorageOptions options = StorageOptions.builder().projectId("project").build();
+ * Storage storage = StorageFactory.instance().get(options);
  * byte[] content = readContent();
  * Blob blob = storage.get("bucket", "blob_name");
  * if (blob == null) {
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
index dbe0eaa19411..29be5b87e08f 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
@@ -16,14 +16,14 @@
 
 package com.google.gcloud.storage;
 
-import static com.google.gcloud.storage.StorageService.PredefinedAcl.PUBLIC_READ;
+import static com.google.gcloud.storage.Storage.PredefinedAcl.PUBLIC_READ;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import com.google.common.collect.Iterables;
-import com.google.gcloud.storage.StorageService.BlobSourceOption;
-import com.google.gcloud.storage.StorageService.BlobTargetOption;
+import com.google.gcloud.storage.Storage.BlobSourceOption;
+import com.google.gcloud.storage.Storage.BlobTargetOption;
 
 import org.junit.Test;
 
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
index 4960c49113ef..d765b0189c96 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -52,26 +52,26 @@ public class SerializationTest {
       Collections.>emptyList());
   private static final ListResult LIST_RESULT =
       new ListResult<>(null, "c", Collections.singletonList(Blob.of("b", "n")));
-  private static StorageService.BlobListOption BLOB_LIST_OPTIONS =
-      StorageService.BlobListOption.maxResults(100);
-  private static StorageService.BlobSourceOption BLOB_SOURCE_OPTIONS =
-      StorageService.BlobSourceOption.generationMatch(1);
-  private static StorageService.BlobTargetOption BLOB_TARGET_OPTIONS =
-      StorageService.BlobTargetOption.generationMatch();
-  private static StorageService.BucketListOption BUCKET_LIST_OPTIONS =
-      StorageService.BucketListOption.prefix("bla");
-  private static StorageService.BucketSourceOption BUCKET_SOURCE_OPTIONS =
-      StorageService.BucketSourceOption.metagenerationMatch(1);
-  private static StorageService.BucketTargetOption BUCKET_TARGET_OPTIONS =
-      StorageService.BucketTargetOption.metagenerationNotMatch();
+  private static Storage.BlobListOption BLOB_LIST_OPTIONS =
+      Storage.BlobListOption.maxResults(100);
+  private static Storage.BlobSourceOption BLOB_SOURCE_OPTIONS =
+      Storage.BlobSourceOption.generationMatch(1);
+  private static Storage.BlobTargetOption BLOB_TARGET_OPTIONS =
+      Storage.BlobTargetOption.generationMatch();
+  private static Storage.BucketListOption BUCKET_LIST_OPTIONS =
+      Storage.BucketListOption.prefix("bla");
+  private static Storage.BucketSourceOption BUCKET_SOURCE_OPTIONS =
+      Storage.BucketSourceOption.metagenerationMatch(1);
+  private static Storage.BucketTargetOption BUCKET_TARGET_OPTIONS =
+      Storage.BucketTargetOption.metagenerationNotMatch();
 
   @Test
   public void testServiceOptions() throws Exception {
-    StorageServiceOptions options = StorageServiceOptions.builder()
+    StorageOptions options = StorageOptions.builder()
         .projectId("p1")
         .authCredentials(AuthCredentials.createForAppEngine())
         .build();
-    StorageServiceOptions serializedCopy = serializeAndDeserialize(options);
+    StorageOptions serializedCopy = serializeAndDeserialize(options);
     assertEquals(options, serializedCopy);
 
     options = options.toBuilder()

From 88cbd26835414d8ae2c48f9ebb85b7a846e088d0 Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 29 May 2015 12:35:36 -0700
Subject: [PATCH 254/732] fix merge conflicts

---
 .../java/com/google/gcloud/examples/StorageExample.java   | 8 ++++----
 .../src/main/java/com/google/gcloud/storage/Storage.java  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
index cd899c32660f..7786c124f6cf 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -86,7 +86,7 @@ private static abstract class StorageAction {
 
     abstract void run(Storage storage, T request) throws Exception;
 
-    abstract T parse(String... args) throws IllegalArgumentException, IOException;
+    abstract T parse(String... args) throws Exception; 
 
     protected String params() {
       return "";
@@ -406,7 +406,7 @@ public String params() {
    */
   private static class ComposeAction extends StorageAction {
     @Override
-    public void run(StorageService storage, ComposeRequest request) {
+    public void run(Storage storage, ComposeRequest request) {
       Blob composedBlob = storage.compose(request);
       System.out.println("Composed " + composedBlob);
     }
@@ -490,12 +490,12 @@ private static class SignUrlAction extends
     private static final char[] PASSWORD =  "notasecret".toCharArray();
 
     @Override
-    public void run(StorageService storage, Tuple tuple)
+    public void run(Storage storage, Tuple tuple)
         throws Exception {
       run(storage, tuple.x(), tuple.y());
     }
 
-    private void run(StorageService storage, ServiceAccountAuthCredentials cred, Blob blob)
+    private void run(Storage storage, ServiceAccountAuthCredentials cred, Blob blob)
         throws IOException {
       Calendar cal = Calendar.getInstance();
       cal.add(Calendar.DATE, 1);
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index fb92cd474d43..fb5cab108f30 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -279,7 +279,7 @@ class ComposeRequest implements Serializable {
 
     private final List sourceBlobs;
     private final Blob target;
-    private final List targetOptions;
+    private final List targetOptions;
 
     public static class SourceBlob implements Serializable {
 

From 2f29ab7a30915c1921546e515ed66c91efc9758e Mon Sep 17 00:00:00 2001
From: aozarov 
Date: Fri, 29 May 2015 14:48:52 -0700
Subject: [PATCH 255/732] rename Bucket to BucketInfo and Blob to BlobInfo

---
 .../gcloud/examples/StorageExample.java       | 138 +++++++--------
 .../google/gcloud/storage/BatchRequest.java   |  26 +--
 .../google/gcloud/storage/BatchResponse.java  |  12 +-
 .../storage/{Blob.java => BlobInfo.java}      |  41 ++---
 .../gcloud/storage/BlobReadChannelImpl.java   |   8 +-
 .../gcloud/storage/BlobWriterChannelImpl.java |   8 +-
 .../storage/{Bucket.java => BucketInfo.java}  |  32 ++--
 .../com/google/gcloud/storage/Storage.java    |  50 +++---
 .../google/gcloud/storage/StorageImpl.java    | 158 +++++++++---------
 .../google/gcloud/storage/package-info.java   |   6 +-
 .../gcloud/storage/BatchRequestTest.java      |  30 ++--
 .../gcloud/storage/BatchResponseTest.java     |  10 +-
 .../{BlobTest.java => BlobInfoTest.java}      |  72 ++++----
 .../{BucketTest.java => BucketInfoTest.java}  |  74 ++++----
 .../gcloud/storage/SerializationTest.java     |  15 +-
 15 files changed, 341 insertions(+), 339 deletions(-)
 rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{Blob.java => BlobInfo.java} (93%)
 rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{Bucket.java => BucketInfo.java} (96%)
 rename gcloud-java-storage/src/test/java/com/google/gcloud/storage/{BlobTest.java => BlobInfoTest.java} (70%)
 rename gcloud-java-storage/src/test/java/com/google/gcloud/storage/{BucketTest.java => BucketInfoTest.java} (72%)

diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
index 7786c124f6cf..76016a1c8790 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -22,10 +22,10 @@
 import com.google.gcloud.spi.StorageRpc.Tuple;
 import com.google.gcloud.storage.BatchRequest;
 import com.google.gcloud.storage.BatchResponse;
-import com.google.gcloud.storage.Blob;
+import com.google.gcloud.storage.BlobInfo;
 import com.google.gcloud.storage.BlobReadChannel;
 import com.google.gcloud.storage.BlobWriteChannel;
-import com.google.gcloud.storage.Bucket;
+import com.google.gcloud.storage.BucketInfo;
 import com.google.gcloud.storage.Storage;
 import com.google.gcloud.storage.Storage.ComposeRequest;
 import com.google.gcloud.storage.Storage.CopyRequest;
@@ -86,21 +86,21 @@ private static abstract class StorageAction {
 
     abstract void run(Storage storage, T request) throws Exception;
 
-    abstract T parse(String... args) throws Exception; 
+    abstract T parse(String... args) throws Exception;
 
     protected String params() {
       return "";
     }
   }
 
-  private static abstract class BlobAction extends StorageAction {
+  private static abstract class BlobAction extends StorageAction {
 
     @Override
-    Blob parse(String... args) {
+    BlobInfo parse(String... args) {
       if (args.length != 2) {
         throw new IllegalArgumentException();
       }
-      return Blob.of(args[0], args[1]);
+      return BlobInfo.of(args[0], args[1]);
     }
 
     @Override
@@ -109,18 +109,18 @@ public String params() {
     }
   }
 
-  private static abstract class BlobsAction extends StorageAction {
+  private static abstract class BlobsAction extends StorageAction {
 
     @Override
-    Blob[] parse(String... args) {
+    BlobInfo[] parse(String... args) {
       if (args.length < 2) {
         throw new IllegalArgumentException();
       }
-      Blob[] blobs = new Blob[args.length - 1];
+      BlobInfo[] blobInfos = new BlobInfo[args.length - 1];
       for (int i = 1; i < args.length; i++) {
-        blobs[i - 1] = Blob.of(args[0], args[i]);
+        blobInfos[i - 1] = BlobInfo.of(args[0], args[i]);
       }
-      return blobs;
+      return blobInfos;
     }
 
     @Override
@@ -138,34 +138,34 @@ public String params() {
    */
   private static class InfoAction extends BlobsAction {
     @Override
-    public void run(Storage storage, Blob... blobs) {
-      if (blobs.length == 1) {
-        if (blobs[0].name().isEmpty()) {
+    public void run(Storage storage, BlobInfo... blobInfos) {
+      if (blobInfos.length == 1) {
+        if (blobInfos[0].name().isEmpty()) {
           // get Bucket
-          Bucket bucket = storage.get(blobs[0].bucket());
-          System.out.println("Bucket info: " + bucket);
+          BucketInfo bucketInfo = storage.get(blobInfos[0].bucket());
+          System.out.println("Bucket info: " + bucketInfo);
         } else {
           // get Blob
-          Blob blob = storage.get(blobs[0].bucket(), blobs[0].name());
-          System.out.println("Blob info: " + blob);
+          BlobInfo blobInfo = storage.get(blobInfos[0].bucket(), blobInfos[0].name());
+          System.out.println("Blob info: " + blobInfo);
         }
       } else {
         // use batch to get multiple blobs.
         BatchRequest.Builder batch = BatchRequest.builder();
-        for (Blob blob : blobs) {
-          batch.get(blob.bucket(), blob.name());
+        for (BlobInfo blobInfo : blobInfos) {
+          batch.get(blobInfo.bucket(), blobInfo.name());
         }
         BatchResponse response = storage.apply(batch.build());
-        for (BatchResponse.Result result : response.gets()) {
+        for (BatchResponse.Result result : response.gets()) {
           System.out.println(result.get());
         }
       }
     }
 
     @Override
-    Blob[] parse(String... args) {
+    BlobInfo[] parse(String... args) {
       if (args.length < 2) {
-        return new Blob[] {Blob.of(args[0], "")};
+        return new BlobInfo[] {BlobInfo.of(args[0], "")};
       }
       return super.parse(args);
     }
@@ -185,24 +185,24 @@ public String params() {
    */
   private static class DeleteAction extends BlobsAction {
     @Override
-    public void run(Storage storage, Blob... blobs) {
-      if (blobs.length == 1) {
-        boolean wasDeleted = storage.delete(blobs[0].bucket(), blobs[0].name());
+    public void run(Storage storage, BlobInfo... blobInfos) {
+      if (blobInfos.length == 1) {
+        boolean wasDeleted = storage.delete(blobInfos[0].bucket(), blobInfos[0].name());
         if (wasDeleted) {
-          System.out.println("Blob " + blobs[0] + " was deleted");
+          System.out.println("Blob " + blobInfos[0] + " was deleted");
         }
       } else {
         // use batch operation
         BatchRequest.Builder batch = BatchRequest.builder();
-        for (Blob blob : blobs) {
-          batch.delete(blob.bucket(), blob.name());
+        for (BlobInfo blobInfo : blobInfos) {
+          batch.delete(blobInfo.bucket(), blobInfo.name());
         }
         int index = 0;
         BatchResponse response = storage.apply(batch.build());
         for (BatchResponse.Result result : response.deletes()) {
           if (result.get()) {
             // request order is maintained
-            System.out.println("Blob " + blobs[index] + " was deleted");
+            System.out.println("Blob " + blobInfos[index] + " was deleted");
           }
           index++;
         }
@@ -232,12 +232,12 @@ String parse(String... args) {
     public void run(Storage storage, String bucket) {
       if (bucket == null) {
         // list buckets
-        for (Bucket b : storage.list()) {
+        for (BucketInfo b : storage.list()) {
           System.out.println(b);
         }
       } else {
         // list a bucket's blobs
-        for (Blob b : storage.list(bucket)) {
+        for (BlobInfo b : storage.list(bucket)) {
           System.out.println(b);
         }
       }
@@ -254,17 +254,17 @@ public String params() {
    *
    * @see Objects: insert
    */
-  private static class UploadAction extends StorageAction> {
+  private static class UploadAction extends StorageAction> {
     @Override
-    public void run(Storage storage, Tuple tuple) throws Exception {
+    public void run(Storage storage, Tuple tuple) throws Exception {
       run(storage, tuple.x(), tuple.y());
     }
 
-    private void run(Storage storage, Path uploadFrom, Blob blob) throws IOException {
+    private void run(Storage storage, Path uploadFrom, BlobInfo blobInfo) throws IOException {
       if (Files.size(uploadFrom) > 1_000_000) {
         // When content is not available or large (1MB or more) it is recommended
         // to write it in chunks via the blob's channel writer.
-        try (BlobWriteChannel writer = storage.writer(blob)) {
+        try (BlobWriteChannel writer = storage.writer(blobInfo)) {
           byte[] buffer = new byte[1024];
           try (InputStream input = Files.newInputStream(uploadFrom)) {
             int limit;
@@ -280,20 +280,20 @@ private void run(Storage storage, Path uploadFrom, Blob blob) throws IOException
       } else {
         byte[] bytes = Files.readAllBytes(uploadFrom);
         // create the blob in one request.
-        storage.create(blob, bytes);
+        storage.create(blobInfo, bytes);
       }
       System.out.println("Blob was created");
     }
 
     @Override
-    Tuple parse(String... args) throws IOException {
+    Tuple parse(String... args) throws IOException {
       if (args.length < 2 || args.length > 3) {
         throw new IllegalArgumentException();
       }
       Path path = Paths.get(args[0]);
       String contentType = Files.probeContentType(path);
       String blob = args.length < 3 ? path.getFileName().toString() : args[2];
-      return Tuple.of(path, Blob.builder(args[1], blob).contentType(contentType).build());
+      return Tuple.of(path, BlobInfo.builder(args[1], blob).contentType(contentType).build());
     }
 
     @Override
@@ -309,17 +309,17 @@ public String params() {
    *
    * @see Objects: get
    */
-  private static class DownloadAction extends StorageAction> {
+  private static class DownloadAction extends StorageAction> {
 
     @Override
-    public void run(Storage storage, Tuple tuple) throws IOException {
+    public void run(Storage storage, Tuple tuple) throws IOException {
       run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y());
     }
 
     private void run(Storage storage, String bucket, String blobName, Path downloadTo)
         throws IOException {
-      Blob blob = storage.get(bucket, blobName);
-      if (blob == null) {
+      BlobInfo blobInfo = storage.get(bucket, blobName);
+      if (blobInfo == null) {
         System.out.println("No such object");
         return;
       }
@@ -327,13 +327,13 @@ private void run(Storage storage, String bucket, String blobName, Path downloadT
       if (downloadTo != null) {
         writeTo = new PrintStream(new FileOutputStream(downloadTo.toFile()));
       }
-      if (blob.size() < 1_000_000) {
+      if (blobInfo.size() < 1_000_000) {
         // Blob is small read all its content in one request
-        byte[] content = storage.load(blob.bucket(), blob.name());
+        byte[] content = storage.load(blobInfo.bucket(), blobInfo.name());
         writeTo.write(content);
       } else {
         // When Blob size is big or unknown use the blob's channel reader.
-        try (BlobReadChannel reader = storage.reader(blob.bucket(), blob.name())) {
+        try (BlobReadChannel reader = storage.reader(blobInfo.bucket(), blobInfo.name())) {
           WritableByteChannel channel = Channels.newChannel(writeTo);
           ByteBuffer bytes = ByteBuffer.allocate(64 * 1024);
           while (reader.read(bytes) > 0) {
@@ -351,7 +351,7 @@ private void run(Storage storage, String bucket, String blobName, Path downloadT
     }
 
     @Override
-    Tuple parse(String... args) {
+    Tuple parse(String... args) {
       if (args.length < 2 || args.length > 3) {
         throw new IllegalArgumentException();
       }
@@ -364,7 +364,7 @@ Tuple parse(String... args) {
       } else {
         path = null;
       }
-      return Tuple.of(Blob.of(args[0], args[1]), path);
+      return Tuple.of(BlobInfo.of(args[0], args[1]), path);
     }
 
     @Override
@@ -381,8 +381,8 @@ public String params() {
   private static class CopyAction extends StorageAction {
     @Override
     public void run(Storage storage, CopyRequest request) {
-      Blob copiedBlob = storage.copy(request);
-      System.out.println("Copied " + copiedBlob);
+      BlobInfo copiedBlobInfo = storage.copy(request);
+      System.out.println("Copied " + copiedBlobInfo);
     }
 
     @Override
@@ -390,7 +390,7 @@ CopyRequest parse(String... args) {
       if (args.length != 4) {
         throw new IllegalArgumentException();
       }
-      return CopyRequest.of(args[0], args[1], Blob.of(args[2], args[3]));
+      return CopyRequest.of(args[0], args[1], BlobInfo.of(args[2], args[3]));
     }
 
     @Override
@@ -407,8 +407,8 @@ public String params() {
   private static class ComposeAction extends StorageAction {
     @Override
     public void run(Storage storage, ComposeRequest request) {
-      Blob composedBlob = storage.compose(request);
-      System.out.println("Composed " + composedBlob);
+      BlobInfo composedBlobInfo = storage.compose(request);
+      System.out.println("Composed " + composedBlobInfo);
     }
 
     @Override
@@ -417,7 +417,7 @@ ComposeRequest parse(String... args) {
         throw new IllegalArgumentException();
       }
       ComposeRequest.Builder request = ComposeRequest.builder();
-      request.target(Blob.of(args[0], args[args.length - 1]));
+      request.target(BlobInfo.of(args[0], args[args.length - 1]));
       for (int i = 1; i < args.length - 1; i++) {
         request.addSource(args[i]);
       }
@@ -435,31 +435,31 @@ public String params() {
    *
    * @see Objects: update
    */
-  private static class UpdateMetadataAction extends StorageAction>> {
+  private static class UpdateMetadataAction extends StorageAction>> {
 
     @Override
-    public void run(Storage storage, Tuple> tuple)
+    public void run(Storage storage, Tuple> tuple)
         throws IOException {
       run(storage, tuple.x().bucket(), tuple.x().name(), tuple.y());
     }
 
     private void run(Storage storage, String bucket, String blobName,
         Map metadata) {
-      Blob blob = storage.get(bucket, blobName);
-      if (blob == null) {
+      BlobInfo blobInfo = storage.get(bucket, blobName);
+      if (blobInfo == null) {
         System.out.println("No such object");
         return;
       }
-      blob = storage.update(blob.toBuilder().metadata(metadata).build());
-      System.out.println("Updated " + blob);
+      blobInfo = storage.update(blobInfo.toBuilder().metadata(metadata).build());
+      System.out.println("Updated " + blobInfo);
     }
 
     @Override
-    Tuple> parse(String... args) {
+    Tuple> parse(String... args) {
       if (args.length < 2) {
         throw new IllegalArgumentException();
       }
-      Blob blob = Blob.of(args[0], args[1]);
+      BlobInfo blobInfo = BlobInfo.of(args[0], args[1]);
       Map metadata = new HashMap<>();
       for (int i = 2; i < args.length; i++) {
         int idx = args[i].indexOf('=');
@@ -469,7 +469,7 @@ Tuple> parse(String... args) {
           metadata.put(args[i].substring(0, idx), args[i].substring(idx + 1));
         }
       }
-      return Tuple.of(blob, metadata);
+      return Tuple.of(blobInfo, metadata);
     }
 
     @Override
@@ -485,27 +485,27 @@ public String params() {
    * @see Signed URLs
    */
   private static class SignUrlAction extends
-      StorageAction> {
+      StorageAction> {
 
     private static final char[] PASSWORD =  "notasecret".toCharArray();
 
     @Override
-    public void run(Storage storage, Tuple tuple)
+    public void run(Storage storage, Tuple tuple)
         throws Exception {
       run(storage, tuple.x(), tuple.y());
     }
 
-    private void run(Storage storage, ServiceAccountAuthCredentials cred, Blob blob)
+    private void run(Storage storage, ServiceAccountAuthCredentials cred, BlobInfo blobInfo)
         throws IOException {
       Calendar cal = Calendar.getInstance();
       cal.add(Calendar.DATE, 1);
       long expiration = cal.getTimeInMillis() / 1000;
       System.out.println("Signed URL: " +
-          storage.signUrl(blob, expiration, SignUrlOption.serviceAccount(cred)));
+          storage.signUrl(blobInfo, expiration, SignUrlOption.serviceAccount(cred)));
     }
 
     @Override
-    Tuple parse(String... args)
+    Tuple parse(String... args)
         throws IOException, KeyStoreException, CertificateException, NoSuchAlgorithmException,
         UnrecoverableKeyException {
       if (args.length != 4) {
@@ -515,7 +515,7 @@ Tuple parse(String... args)
       keystore.load(Files.newInputStream(Paths.get(args[0])), PASSWORD);
       PrivateKey privateKey = (PrivateKey) keystore.getKey("privatekey", PASSWORD);
       ServiceAccountAuthCredentials cred = AuthCredentials.createFor(args[1], privateKey);
-      return Tuple.of(cred, Blob.of(args[2], args[3]));
+      return Tuple.of(cred, BlobInfo.of(args[2], args[3]));
     }
 
     @Override
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
index 5cea321b0071..6959388f34ef 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchRequest.java
@@ -33,15 +33,15 @@ public final class BatchRequest implements Serializable {
 
   private static final long serialVersionUID = -1527992265939800345L;
 
-  private final Map> toDelete;
-  private final Map> toUpdate;
-  private final Map> toGet;
+  private final Map> toDelete;
+  private final Map> toUpdate;
+  private final Map> toGet;
 
   public static class Builder {
 
-    private Map> toDelete = new LinkedHashMap<>();
-    private Map> toUpdate = new LinkedHashMap<>();
-    private Map> toGet = new LinkedHashMap<>();
+    private Map> toDelete = new LinkedHashMap<>();
+    private Map> toUpdate = new LinkedHashMap<>();
+    private Map> toGet = new LinkedHashMap<>();
 
     private Builder() {}
 
@@ -49,15 +49,15 @@ private Builder() {}
      * Delete the given blob.
      */
     public Builder delete(String bucket, String blob, BlobSourceOption... options) {
-      toDelete.put(Blob.of(bucket, blob), Lists.newArrayList(options));
+      toDelete.put(BlobInfo.of(bucket, blob), Lists.newArrayList(options));
       return this;
     }
 
     /**
      * Update the given blob.
      */
-    public Builder update(Blob blob, BlobTargetOption... options) {
-      toUpdate.put(blob, Lists.newArrayList(options));
+    public Builder update(BlobInfo blobInfo, BlobTargetOption... options) {
+      toUpdate.put(blobInfo, Lists.newArrayList(options));
       return this;
     }
 
@@ -65,7 +65,7 @@ public Builder update(Blob blob, BlobTargetOption... options) {
      * Retrieve metadata for the given blob.
      */
     public Builder get(String bucket, String blob, BlobSourceOption... options) {
-      toGet.put(Blob.of(bucket, blob), Lists.newArrayList(options));
+      toGet.put(BlobInfo.of(bucket, blob), Lists.newArrayList(options));
       return this;
     }
 
@@ -96,15 +96,15 @@ public boolean equals(Object obj) {
         && Objects.equals(toGet, other.toGet);
   }
 
-  public Map> toDelete() {
+  public Map> toDelete() {
     return toDelete;
   }
 
-  public Map> toUpdate() {
+  public Map> toUpdate() {
     return toUpdate;
   }
 
-  public Map> toGet() {
+  public Map> toGet() {
     return toGet;
   }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
index 1a05fba819c4..45aa1674b03c 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -31,8 +31,8 @@ public final class BatchResponse implements Serializable {
   private static final long serialVersionUID = 1057416839397037706L;
 
   private final List> deleteResult;
-  private final List> updateResult;
-  private final List> getResult;
+  private final List> updateResult;
+  private final List> getResult;
 
   public static class Result implements Serializable {
 
@@ -112,8 +112,8 @@ static  Result empty() {
     }
   }
 
-  public BatchResponse(List> deleteResult, List> updateResult,
-      List> getResult) {
+  public BatchResponse(List> deleteResult, List> updateResult,
+      List> getResult) {
     this.deleteResult = ImmutableList.copyOf(deleteResult);
     this.updateResult = ImmutableList.copyOf(updateResult);
     this.getResult = ImmutableList.copyOf(getResult);
@@ -145,14 +145,14 @@ public List> deletes() {
   /**
    * Returns the results for the update operations using the request order.
    */
-  public List> updates() {
+  public List> updates() {
     return updateResult;
   }
 
   /**
    * Returns the results for the get operations using the request order.
    */
-  public List> gets() {
+  public List> gets() {
     return getResult;
   }
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java
similarity index 93%
rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java
index 31b88981187d..3ef01af8d1f3 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java
@@ -41,24 +41,25 @@
  *
  * @see Concepts and Terminology
  */
-public final class Blob implements Serializable {
+public final class BlobInfo implements Serializable {
 
   private static final long serialVersionUID = 2228487739943277159L;
 
-  static final Function FROM_PB_FUNCTION =
-      new Function() {
+  static final Function FROM_PB_FUNCTION =
+      new Function() {
         @Override
-        public Blob apply(StorageObject pb) {
-          return Blob.fromPb(pb);
+        public BlobInfo apply(StorageObject pb) {
+          return BlobInfo.fromPb(pb);
         }
       };
 
-  static final Function TO_PB_FUNCTION = new Function() {
-    @Override
-    public StorageObject apply(Blob blob) {
-      return blob.toPb();
-    }
-  };
+  static final Function TO_PB_FUNCTION =
+      new Function() {
+        @Override
+        public StorageObject apply(BlobInfo blobInfo) {
+          return blobInfo.toPb();
+        }
+      };
 
   private final String bucket;
   private final String id;
@@ -220,14 +221,14 @@ Builder updateTime(Long updateTime) {
       return this;
     }
 
-    public Blob build() {
+    public BlobInfo build() {
       checkNotNull(bucket);
       checkNotNull(name);
-      return new Blob(this);
+      return new BlobInfo(this);
     }
   }
 
-  private Blob(Builder builder) {
+  private BlobInfo(Builder builder) {
     bucket = builder.bucket;
     name = builder.name;
     id = builder.id;
@@ -377,12 +378,12 @@ public String toString() {
         .toString();
   }
 
-  public static Blob of(String bucket, String name) {
+  public static BlobInfo of(String bucket, String name) {
     return builder(bucket, name).build();
   }
 
-  public static Builder builder(Bucket bucket, String name) {
-    return builder(bucket.name(), name);
+  public static Builder builder(BucketInfo bucketInfo, String name) {
+    return builder(bucketInfo.name(), name);
   }
 
   public static Builder builder(String bucket, String name) {
@@ -396,10 +397,10 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (!(obj instanceof Blob)) {
+    if (!(obj instanceof BlobInfo)) {
       return  false;
     }
-    return Objects.equals(toPb(), ((Blob) obj).toPb());
+    return Objects.equals(toPb(), ((BlobInfo) obj).toPb());
   }
 
   StorageObject toPb() {
@@ -444,7 +445,7 @@ public ObjectAccessControl apply(Acl acl) {
     return storageObject;
   }
 
-  static Blob fromPb(StorageObject storageObject) {
+  static BlobInfo fromPb(StorageObject storageObject) {
     Builder builder = new Builder()
         .bucket(storageObject.getBucket())
         .cacheControl(storageObject.getCacheControl())
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
index 5a5d165751c4..9d1d37f93ab1 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobReadChannelImpl.java
@@ -37,7 +37,7 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private static final long serialVersionUID = 4821762590742862669L;
 
   private final StorageOptions serviceOptions;
-  private final Blob blob;
+  private final BlobInfo blobInfo;
   private final Map requestOptions;
   private int position;
   private boolean isOpen;
@@ -49,10 +49,10 @@ class BlobReadChannelImpl implements BlobReadChannel {
   private transient int bufferPos;
   private transient byte[] buffer;
 
-  BlobReadChannelImpl(StorageOptions serviceOptions, Blob blob,
+  BlobReadChannelImpl(StorageOptions serviceOptions, BlobInfo blobInfo,
       Map requestOptions) {
     this.serviceOptions = serviceOptions;
-    this.blob = blob;
+    this.blobInfo = blobInfo;
     this.requestOptions = requestOptions;
     isOpen = true;
     initTransients();
@@ -75,7 +75,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
 
   private void initTransients() {
     storageRpc = serviceOptions.storageRpc();
-    storageObject = blob.toPb();
+    storageObject = blobInfo.toPb();
   }
 
   @Override
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
index 5e41ed00fc54..27cd807f043f 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
@@ -39,7 +39,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private static final int DEFAULT_CHUNK_SIZE = 8 * MIN_CHUNK_SIZE;
 
   private final StorageOptions options;
-  private final Blob blob;
+  private final BlobInfo blobInfo;
   private final String uploadId;
   private int position;
   private byte[] buffer = new byte[0];
@@ -50,10 +50,10 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
 
-  public BlobWriterChannelImpl(StorageOptions options, Blob blob,
+  public BlobWriterChannelImpl(StorageOptions options, BlobInfo blobInfo,
       Map optionsMap) {
     this.options = options;
-    this.blob = blob;
+    this.blobInfo = blobInfo;
     initTransients();
     uploadId = options.storageRpc().open(storageObject, optionsMap);
   }
@@ -91,7 +91,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
 
   private void initTransients() {
     storageRpc = options.storageRpc();
-    storageObject = blob.toPb();
+    storageObject = blobInfo.toPb();
   }
 
   private void validateOpen() throws IOException {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
similarity index 96%
rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
index 85d601487c28..60926e01dbb2 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Bucket.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -48,7 +48,7 @@
  *
  * @see Concepts and Terminology
  */
-public final class Bucket implements Serializable {
+public final class BucketInfo implements Serializable {
 
   private static final long serialVersionUID = -3946094202176916586L;
 
@@ -69,19 +69,19 @@ public final class Bucket implements Serializable {
   private final Location location;
   private final StorageClass storageClass;
 
-  static final Function FROM_PB_FUNCTION =
-      new Function() {
+  static final Function FROM_PB_FUNCTION =
+      new Function() {
         @Override
-        public Bucket apply(com.google.api.services.storage.model.Bucket pb) {
-          return Bucket.fromPb(pb);
+        public BucketInfo apply(com.google.api.services.storage.model.Bucket pb) {
+          return BucketInfo.fromPb(pb);
         }
       };
 
-  static final Function TO_PB_FUNCTION =
-      new Function() {
+  static final Function TO_PB_FUNCTION =
+      new Function() {
         @Override
-        public com.google.api.services.storage.model.Bucket apply(Bucket bucket) {
-          return bucket.toPb();
+        public com.google.api.services.storage.model.Bucket apply(BucketInfo bucketInfo) {
+          return bucketInfo.toPb();
         }
       };
 
@@ -491,13 +491,13 @@ public Builder defaultAcl(Iterable acl) {
       return this;
     }
 
-    public Bucket build() {
+    public BucketInfo build() {
       checkNotNull(name);
-      return new Bucket(this);
+      return new BucketInfo(this);
     }
   }
 
-  private Bucket(Builder builder) {
+  private BucketInfo(Builder builder) {
     id = builder.id;
     name = builder.name;
     etag = builder.etag;
@@ -607,10 +607,10 @@ public int hashCode() {
 
   @Override
   public boolean equals(Object obj) {
-    if (!(obj instanceof Bucket)) {
+    if (!(obj instanceof BucketInfo)) {
       return  false;
     }
-    return Objects.equals(toPb(), ((Bucket) obj).toPb());
+    return Objects.equals(toPb(), ((BucketInfo) obj).toPb());
   }
 
   @Override
@@ -620,7 +620,7 @@ public String toString() {
         .toString();
   }
 
-  public static Bucket of(String name) {
+  public static BucketInfo of(String name) {
     return builder(name).build();
   }
 
@@ -691,7 +691,7 @@ public Rule apply(DeleteRule deleteRule) {
     return bucketPb;
   }
 
-  static Bucket fromPb(com.google.api.services.storage.model.Bucket bucketPb) {
+  static BucketInfo fromPb(com.google.api.services.storage.model.Bucket bucketPb) {
     Builder builder = new Builder()
         .name(bucketPb.getName())
         .id(bucketPb.getId())
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index fb5cab108f30..770a5a924ff6 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -278,7 +278,7 @@ class ComposeRequest implements Serializable {
     private static final long serialVersionUID = -7385681353748590911L;
 
     private final List sourceBlobs;
-    private final Blob target;
+    private final BlobInfo target;
     private final List targetOptions;
 
     public static class SourceBlob implements Serializable {
@@ -309,7 +309,7 @@ public Long generation() {
     public static class Builder {
 
       private final List sourceBlobs = new LinkedList<>();
-      private Blob target;
+      private BlobInfo target;
       private final Set targetOptions = new LinkedHashSet<>();
 
       public Builder addSource(Iterable blobs) {
@@ -331,7 +331,7 @@ public Builder addSource(String blob, long generation) {
         return this;
       }
 
-      public Builder target(Blob target) {
+      public Builder target(BlobInfo target) {
         this.target = target;
         return this;
       }
@@ -358,7 +358,7 @@ public List sourceBlobs() {
       return sourceBlobs;
     }
 
-    public Blob target() {
+    public BlobInfo target() {
       return target;
     }
 
@@ -366,12 +366,12 @@ public List targetOptions() {
       return targetOptions;
     }
 
-    public static ComposeRequest of(Iterable sources, Blob target) {
+    public static ComposeRequest of(Iterable sources, BlobInfo target) {
       return builder().target(target).addSource(sources).build();
     }
 
     public static ComposeRequest of(String bucket, Iterable sources, String target) {
-      return of(sources, Blob.of(bucket, target));
+      return of(sources, BlobInfo.of(bucket, target));
     }
 
     public static Builder builder() {
@@ -386,7 +386,7 @@ class CopyRequest implements Serializable {
     private final String sourceBucket;
     private final String sourceBlob;
     private final List sourceOptions;
-    private final Blob target;
+    private final BlobInfo target;
     private final List targetOptions;
 
     public static class Builder {
@@ -394,7 +394,7 @@ public static class Builder {
       private String sourceBucket;
       private String sourceBlob;
       private final Set sourceOptions = new LinkedHashSet<>();
-      private Blob target;
+      private BlobInfo target;
       private final Set targetOptions = new LinkedHashSet<>();
 
       public Builder source(String bucket, String blob) {
@@ -408,7 +408,7 @@ public Builder sourceOptions(BlobSourceOption... options) {
         return this;
       }
 
-      public Builder target(Blob target) {
+      public Builder target(BlobInfo target) {
         this.target = target;
         return this;
       }
@@ -446,7 +446,7 @@ public List sourceOptions() {
       return sourceOptions;
     }
 
-    public Blob target() {
+    public BlobInfo target() {
       return target;
     }
 
@@ -454,12 +454,12 @@ public List targetOptions() {
       return targetOptions;
     }
 
-    public static CopyRequest of(String sourceBucket, String sourceBlob, Blob target) {
+    public static CopyRequest of(String sourceBucket, String sourceBlob, BlobInfo target) {
       return builder().source(sourceBucket, sourceBlob).target(target).build();
     }
 
     public static CopyRequest of(String sourceBucket, String sourceBlob, String targetBlob) {
-      return of(sourceBucket, sourceBlob, Blob.of(sourceBucket, targetBlob));
+      return of(sourceBucket, sourceBlob, BlobInfo.of(sourceBucket, targetBlob));
     }
 
     public static Builder builder() {
@@ -473,7 +473,7 @@ public static Builder builder() {
    * @return a complete bucket information.
    * @throws StorageException upon failure
    */
-  Bucket create(Bucket bucket, BucketTargetOption... options);
+  BucketInfo create(BucketInfo bucketInfo, BucketTargetOption... options);
 
   /**
    * Create a new blob.
@@ -481,35 +481,35 @@ public static Builder builder() {
    * @return a complete blob information.
    * @throws StorageException upon failure
    */
-  Blob create(Blob blob, byte[] content, BlobTargetOption... options);
+  BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options);
 
   /**
    * Return the requested bucket or {@code null} if not found.
    *
    * @throws StorageException upon failure
    */
-  Bucket get(String bucket, BucketSourceOption... options);
+  BucketInfo get(String bucket, BucketSourceOption... options);
 
   /**
    * Return the requested blob or {@code null} if not found.
    *
    * @throws StorageException upon failure
    */
-  Blob get(String bucket, String blob, BlobSourceOption... options);
+  BlobInfo get(String bucket, String blob, BlobSourceOption... options);
 
   /**
    * List the project's buckets.
    *
    * @throws StorageException upon failure
    */
-  ListResult list(BucketListOption... options);
+  ListResult list(BucketListOption... options);
 
   /**
    * List the bucket's blobs.
    *
    * @throws StorageException upon failure
    */
-  ListResult list(String bucket, BlobListOption... options);
+  ListResult list(String bucket, BlobListOption... options);
 
   /**
    * Update bucket information.
@@ -517,7 +517,7 @@ public static Builder builder() {
    * @return the updated bucket
    * @throws StorageException upon failure
    */
-  Bucket update(Bucket bucket, BucketTargetOption... options);
+  BucketInfo update(BucketInfo bucketInfo, BucketTargetOption... options);
 
   /**
    * Update blob information.
@@ -525,7 +525,7 @@ public static Builder builder() {
    * @return the updated blob
    * @throws StorageException upon failure
    */
-  Blob update(Blob blob, BlobTargetOption... options);
+  BlobInfo update(BlobInfo blobInfo, BlobTargetOption... options);
 
   /**
    * Delete the requested bucket.
@@ -549,7 +549,7 @@ public static Builder builder() {
    * @return the composed blob.
    * @throws StorageException upon failure
    */
-  Blob compose(ComposeRequest composeRequest);
+  BlobInfo compose(ComposeRequest composeRequest);
 
   /**
    * Send a copy request.
@@ -557,7 +557,7 @@ public static Builder builder() {
    * @return the copied blob.
    * @throws StorageException upon failure
    */
-  Blob copy(CopyRequest copyRequest);
+  BlobInfo copy(CopyRequest copyRequest);
 
   /**
    * Load the content of the given blob.
@@ -587,7 +587,7 @@ public static Builder builder() {
    *
    * @throws StorageException upon failure
    */
-  BlobWriteChannel writer(Blob blob, BlobTargetOption... options);
+  BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options);
 
   /**
    * Generates a signed URL for a blob.
@@ -597,9 +597,9 @@ public static Builder builder() {
    * This is particularly useful if you don't want publicly
    * accessible blobs, but don't want to require users to explicitly log in.
    *
-   * @param blob the blob associated with the signed url
+   * @param blobInfo the blob associated with the signed url
    * @param  expirationTimeInSeconds the signed URL expiration (using epoch time)
    * @see Signed-URLs
    */
-  URL signUrl(Blob blob, long expirationTimeInSeconds, SignUrlOption... options);
+  URL signUrl(BlobInfo blobInfo, long expirationTimeInSeconds, SignUrlOption... options);
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index 29db26481935..44c940f1c64a 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -99,10 +99,10 @@ public RetryResult beforeEval(Exception exception) {
   }
 
   @Override
-  public Bucket create(Bucket bucket, BucketTargetOption... options) {
-    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(bucket, options);
-    return Bucket.fromPb(runWithRetries(
+  public BucketInfo create(BucketInfo bucketInfo, BucketTargetOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb();
+    final Map optionsMap = optionMap(bucketInfo, options);
+    return BucketInfo.fromPb(runWithRetries(
         new Callable() {
           @Override
           public com.google.api.services.storage.model.Bucket call() {
@@ -112,10 +112,10 @@ public com.google.api.services.storage.model.Bucket call() {
   }
 
   @Override
-  public Blob create(Blob blob, final byte[] content, BlobTargetOption... options) {
-    final StorageObject blobPb = blob.toPb();
-    final Map optionsMap = optionMap(blob, options);
-    return Blob.fromPb(runWithRetries(new Callable() {
+  public BlobInfo create(BlobInfo blobInfo, final byte[] content, BlobTargetOption... options) {
+    final StorageObject blobPb = blobInfo.toPb();
+    final Map optionsMap = optionMap(blobInfo, options);
+    return BlobInfo.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
         return storageRpc.create(blobPb, firstNonNull(content, EMPTY_BYTE_ARRAY), optionsMap);
@@ -124,8 +124,8 @@ public StorageObject call() {
   }
 
   @Override
-  public Bucket get(String bucket, BucketSourceOption... options) {
-    final com.google.api.services.storage.model.Bucket bucketPb = Bucket.of(bucket).toPb();
+  public BucketInfo get(String bucket, BucketSourceOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb();
     final Map optionsMap = optionMap(options);
     com.google.api.services.storage.model.Bucket answer = runWithRetries(
         new Callable() {
@@ -141,12 +141,12 @@ public com.google.api.services.storage.model.Bucket call() {
             }
           }
         }, options().retryParams(), EXCEPTION_HANDLER);
-    return answer == null ? null : Bucket.fromPb(answer);
+    return answer == null ? null : BucketInfo.fromPb(answer);
   }
 
   @Override
-  public Blob get(String bucket, String blob, BlobSourceOption... options) {
-    final StorageObject storedObject = Blob.of(bucket, blob).toPb();
+  public BlobInfo get(String bucket, String blob, BlobSourceOption... options) {
+    final StorageObject storedObject = BlobInfo.of(bucket, blob).toPb();
     final Map optionsMap = optionMap(options);
     StorageObject storageObject = runWithRetries(new Callable() {
       @Override
@@ -161,7 +161,7 @@ public StorageObject call() {
         }
       }
     }, options().retryParams(), EXCEPTION_HANDLER);
-    return storageObject == null ? null : Blob.fromPb(storageObject);
+    return storageObject == null ? null : BlobInfo.fromPb(storageObject);
   }
 
   private static abstract class BasePageFetcher
@@ -185,7 +185,7 @@ private static abstract class BasePageFetcher
     }
   }
 
-  private static class BucketPageFetcher extends BasePageFetcher {
+  private static class BucketPageFetcher extends BasePageFetcher {
 
     private static final long serialVersionUID = -5490616010200159174L;
 
@@ -195,12 +195,12 @@ private static class BucketPageFetcher extends BasePageFetcher {
     }
 
     @Override
-    public ListResult nextPage() {
+    public ListResult nextPage() {
       return listBuckets(serviceOptions, requestOptions);
     }
   }
 
-  private static class BlobPageFetcher extends BasePageFetcher {
+  private static class BlobPageFetcher extends BasePageFetcher {
 
     private static final long serialVersionUID = -5490616010200159174L;
     private final String bucket;
@@ -212,17 +212,17 @@ private static class BlobPageFetcher extends BasePageFetcher {
     }
 
     @Override
-    public ListResult nextPage() {
+    public ListResult nextPage() {
       return listBlobs(bucket, serviceOptions, requestOptions);
     }
   }
 
   @Override
-  public ListResult list(BucketListOption... options) {
+  public ListResult list(BucketListOption... options) {
     return listBuckets(options(), optionMap(options));
   }
 
-  private static ListResult listBuckets(final StorageOptions serviceOptions,
+  private static ListResult listBuckets(final StorageOptions serviceOptions,
       final Map optionsMap) {
     Tuple> result = runWithRetries(
         new Callable>>() {
@@ -234,20 +234,20 @@ public Tuple> cal
     String cursor = result.x();
     return new ListResult<>(new BucketPageFetcher(serviceOptions, cursor, optionsMap), cursor,
         Iterables.transform(result.y(),
-            new Function() {
+            new Function() {
               @Override
-              public Bucket apply(com.google.api.services.storage.model.Bucket bucketPb) {
-                return Bucket.fromPb(bucketPb);
+              public BucketInfo apply(com.google.api.services.storage.model.Bucket bucketPb) {
+                return BucketInfo.fromPb(bucketPb);
               }
             }));
   }
 
   @Override
-  public ListResult list(final String bucket, BlobListOption... options) {
+  public ListResult list(final String bucket, BlobListOption... options) {
     return listBlobs(bucket, options(), optionMap(options));
   }
 
-  private static ListResult listBlobs(final String bucket,
+  private static ListResult listBlobs(final String bucket,
       final StorageOptions serviceOptions, final Map optionsMap) {
     Tuple> result = runWithRetries(
         new Callable>>() {
@@ -259,19 +259,19 @@ public Tuple> call() {
     String cursor = result.x();
     return new ListResult<>(new BlobPageFetcher(bucket, serviceOptions, cursor, optionsMap), cursor,
         Iterables.transform(result.y(),
-            new Function() {
+            new Function() {
               @Override
-              public Blob apply(StorageObject storageObject) {
-                return Blob.fromPb(storageObject);
+              public BlobInfo apply(StorageObject storageObject) {
+                return BlobInfo.fromPb(storageObject);
               }
             }));
   }
 
   @Override
-  public Bucket update(Bucket bucket, BucketTargetOption... options) {
-    final com.google.api.services.storage.model.Bucket bucketPb = bucket.toPb();
-    final Map optionsMap = optionMap(bucket, options);
-    return Bucket.fromPb(runWithRetries(
+  public BucketInfo update(BucketInfo bucketInfo, BucketTargetOption... options) {
+    final com.google.api.services.storage.model.Bucket bucketPb = bucketInfo.toPb();
+    final Map optionsMap = optionMap(bucketInfo, options);
+    return BucketInfo.fromPb(runWithRetries(
         new Callable() {
           @Override
           public com.google.api.services.storage.model.Bucket call() {
@@ -281,10 +281,10 @@ public com.google.api.services.storage.model.Bucket call() {
   }
 
   @Override
-  public Blob update(Blob blob, BlobTargetOption... options) {
-    final StorageObject storageObject = blob.toPb();
-    final Map optionsMap = optionMap(blob, options);
-    return Blob.fromPb(runWithRetries(new Callable() {
+  public BlobInfo update(BlobInfo blobInfo, BlobTargetOption... options) {
+    final StorageObject storageObject = blobInfo.toPb();
+    final Map optionsMap = optionMap(blobInfo, options);
+    return BlobInfo.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
         return storageRpc.patch(storageObject, optionsMap);
@@ -294,7 +294,7 @@ public StorageObject call() {
 
   @Override
   public boolean delete(String bucket, BucketSourceOption... options) {
-    final com.google.api.services.storage.model.Bucket bucketPb = Bucket.of(bucket).toPb();
+    final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb();
     final Map optionsMap = optionMap(options);
     return runWithRetries(new Callable() {
       @Override
@@ -306,7 +306,7 @@ public Boolean call() {
 
   @Override
   public boolean delete(String bucket, String blob, BlobSourceOption... options) {
-    final StorageObject storageObject = Blob.of(bucket, blob).toPb();
+    final StorageObject storageObject = BlobInfo.of(bucket, blob).toPb();
     final Map optionsMap = optionMap(options);
     return runWithRetries(new Callable() {
       @Override
@@ -317,17 +317,17 @@ public Boolean call() {
   }
 
   @Override
-  public Blob compose(final ComposeRequest composeRequest) {
+  public BlobInfo compose(final ComposeRequest composeRequest) {
     final List sources =
         Lists.newArrayListWithCapacity(composeRequest.sourceBlobs().size());
     for (ComposeRequest.SourceBlob sourceBlob : composeRequest.sourceBlobs()) {
-      sources.add(Blob.builder(composeRequest.target().bucket(), sourceBlob.name())
+      sources.add(BlobInfo.builder(composeRequest.target().bucket(), sourceBlob.name())
           .generation(sourceBlob.generation()).build().toPb());
     }
     final StorageObject target = composeRequest.target().toPb();
     final Map targetOptions = optionMap(composeRequest.target().generation(),
         composeRequest.target().metageneration(), composeRequest.targetOptions());
-    return Blob.fromPb(runWithRetries(new Callable() {
+    return BlobInfo.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
         return storageRpc.compose(sources, target, targetOptions);
@@ -336,16 +336,16 @@ public StorageObject call() {
   }
 
   @Override
-  public Blob copy(CopyRequest copyRequest) {
+  public BlobInfo copy(CopyRequest copyRequest) {
     final StorageObject source =
-        Blob.of(copyRequest.sourceBucket(), copyRequest.sourceBlob()).toPb();
+        BlobInfo.of(copyRequest.sourceBucket(), copyRequest.sourceBlob()).toPb();
     copyRequest.sourceOptions();
     final Map sourceOptions =
         optionMap(null, null, copyRequest.sourceOptions(), true);
     final StorageObject target = copyRequest.target().toPb();
     final Map targetOptions = optionMap(copyRequest.target().generation(),
         copyRequest.target().metageneration(), copyRequest.targetOptions());
-    return Blob.fromPb(runWithRetries(new Callable() {
+    return BlobInfo.fromPb(runWithRetries(new Callable() {
       @Override
       public StorageObject call() {
         return storageRpc.copy(source, sourceOptions, target, targetOptions);
@@ -355,7 +355,7 @@ public StorageObject call() {
 
   @Override
   public byte[] load(String bucket, String blob, BlobSourceOption... options) {
-    final StorageObject storageObject = Blob.of(bucket, blob).toPb();
+    final StorageObject storageObject = BlobInfo.of(bucket, blob).toPb();
     final Map optionsMap = optionMap(options);
     return runWithRetries(new Callable() {
       @Override
@@ -369,37 +369,37 @@ public byte[] call() {
   public BatchResponse apply(BatchRequest batchRequest) {
     List>> toDelete =
         Lists.newArrayListWithCapacity(batchRequest.toDelete().size());
-    for (Map.Entry> entry : batchRequest.toDelete().entrySet()) {
-      Blob blob = entry.getKey();
+    for (Map.Entry> entry : batchRequest.toDelete().entrySet()) {
+      BlobInfo blobInfo = entry.getKey();
       Map optionsMap =
-          optionMap(blob.generation(), blob.metageneration(), entry.getValue());
-      StorageObject storageObject = blob.toPb();
+          optionMap(blobInfo.generation(), blobInfo.metageneration(), entry.getValue());
+      StorageObject storageObject = blobInfo.toPb();
       toDelete.add(Tuple.>of(storageObject, optionsMap));
     }
     List>> toUpdate =
         Lists.newArrayListWithCapacity(batchRequest.toUpdate().size());
-    for (Map.Entry> entry : batchRequest.toUpdate().entrySet()) {
-      Blob blob = entry.getKey();
+    for (Map.Entry> entry : batchRequest.toUpdate().entrySet()) {
+      BlobInfo blobInfo = entry.getKey();
       Map optionsMap =
-          optionMap(blob.generation(), blob.metageneration(), entry.getValue());
-      toUpdate.add(Tuple.>of(blob.toPb(), optionsMap));
+          optionMap(blobInfo.generation(), blobInfo.metageneration(), entry.getValue());
+      toUpdate.add(Tuple.>of(blobInfo.toPb(), optionsMap));
     }
     List>> toGet =
         Lists.newArrayListWithCapacity(batchRequest.toGet().size());
-    for (Map.Entry> entry : batchRequest.toGet().entrySet()) {
-      Blob blob = entry.getKey();
+    for (Map.Entry> entry : batchRequest.toGet().entrySet()) {
+      BlobInfo blobInfo = entry.getKey();
       Map optionsMap =
-          optionMap(blob.generation(), blob.metageneration(), entry.getValue());
-      toGet.add(Tuple.>of(blob.toPb(), optionsMap));
+          optionMap(blobInfo.generation(), blobInfo.metageneration(), entry.getValue());
+      toGet.add(Tuple.>of(blobInfo.toPb(), optionsMap));
     }
     StorageRpc.BatchResponse response =
         storageRpc.batch(new StorageRpc.BatchRequest(toDelete, toUpdate, toGet));
     List> deletes = transformBatchResult(
         toDelete, response.deletes, Functions.identity());
-    List> updates = transformBatchResult(
-        toUpdate, response.updates, Blob.FROM_PB_FUNCTION);
-    List> gets = transformBatchResult(
-        toGet, response.gets, Blob.FROM_PB_FUNCTION, HTTP_NOT_FOUND);
+    List> updates = transformBatchResult(
+        toUpdate, response.updates, BlobInfo.FROM_PB_FUNCTION);
+    List> gets = transformBatchResult(
+        toGet, response.gets, BlobInfo.FROM_PB_FUNCTION, HTTP_NOT_FOUND);
     return new BatchResponse(deletes, updates, gets);
   }
 
@@ -429,17 +429,17 @@ private  List> transformBatch
   @Override
   public BlobReadChannel reader(String bucket, String blob, BlobSourceOption... options) {
     Map optionsMap = optionMap(options);
-    return new BlobReadChannelImpl(options(), Blob.of(bucket, blob), optionsMap);
+    return new BlobReadChannelImpl(options(), BlobInfo.of(bucket, blob), optionsMap);
   }
 
  @Override
-  public BlobWriteChannel writer(Blob blob, BlobTargetOption... options) {
-    final Map optionsMap = optionMap(blob, options);
-    return new BlobWriterChannelImpl(options(), blob, optionsMap);
+  public BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) {
+    final Map optionsMap = optionMap(blobInfo, options);
+    return new BlobWriterChannelImpl(options(), blobInfo, optionsMap);
   }
 
   @Override
-  public URL signUrl(Blob blob, long expiration, SignUrlOption... options) {
+  public URL signUrl(BlobInfo blobInfo, long expiration, SignUrlOption... options) {
     EnumMap optionMap = Maps.newEnumMap(SignUrlOption.Option.class);
     for (SignUrlOption option : options) {
       optionMap.put(option.option(), option.value());
@@ -460,28 +460,28 @@ public URL signUrl(Blob blob, long expiration, SignUrlOption... options) {
     }
     stBuilder.append('\n');
     if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.MD5) , false)) {
-      checkArgument(blob.md5() != null, "Blob is missing a value for md5");
-      stBuilder.append(blob.md5());
+      checkArgument(blobInfo.md5() != null, "Blob is missing a value for md5");
+      stBuilder.append(blobInfo.md5());
     }
     stBuilder.append('\n');
     if (firstNonNull((Boolean) optionMap.get(SignUrlOption.Option.CONTENT_TYPE) , false)) {
-      checkArgument(blob.contentType() != null, "Blob is missing a value for content-type");
-      stBuilder.append(blob.contentType());
+      checkArgument(blobInfo.contentType() != null, "Blob is missing a value for content-type");
+      stBuilder.append(blobInfo.contentType());
     }
     stBuilder.append('\n');
     stBuilder.append(expiration).append('\n');
     StringBuilder path = new StringBuilder();
-    if (!blob.bucket().startsWith("/")) {
+    if (!blobInfo.bucket().startsWith("/")) {
       path.append('/');
     }
-    path.append(blob.bucket());
-    if (!blob.bucket().endsWith("/")) {
+    path.append(blobInfo.bucket());
+    if (!blobInfo.bucket().endsWith("/")) {
       path.append('/');
     }
-    if (blob.name().startsWith("/")) {
+    if (blobInfo.name().startsWith("/")) {
       path.setLength(stBuilder.length() - 1);
     }
-    path.append(blob.name());
+    path.append(blobInfo.name());
     stBuilder.append(path);
     try {
       Signature signer = Signature.getInstance("SHA256withRSA");
@@ -558,11 +558,11 @@ private static  void addToOptionMap(StorageRpc.Option getOption, StorageRpc.O
     return optionMap(generation, metaGeneration, Arrays.asList(options));
   }
 
-  private Map optionMap(Bucket bucket, Option... options) {
-    return optionMap(null, bucket.metageneration(), options);
+  private Map optionMap(BucketInfo bucketInfo, Option... options) {
+    return optionMap(null, bucketInfo.metageneration(), options);
   }
 
-  private Map optionMap(Blob blob, Option... options) {
-    return optionMap(blob.generation(), blob.metageneration(), options);
+  private Map optionMap(BlobInfo blobInfo, Option... options) {
+    return optionMap(blobInfo.generation(), blobInfo.metageneration(), options);
   }
 }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
index c648ea88e39d..f689d264a828 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
@@ -22,9 +22,9 @@
  * StorageOptions options = StorageOptions.builder().projectId("project").build();
  * Storage storage = StorageFactory.instance().get(options);
  * byte[] content = readContent();
- * Blob blob = storage.get("bucket", "blob_name");
- * if (blob == null) {
- *   storage.create(Blob.of("bucket", "blob_name"), content);
+ * BlobInfo blobInfo = storage.get("bucket", "blob_name");
+ * if (blobInfo == null) {
+ *   storage.create(BlobInfo.of("bucket", "blob_name"), content);
  * } else {
  *   byte[] prevContent = storage.load("bucket", "blob_name");
  *   content = mergeContent(prevContent, content);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
index 29be5b87e08f..9cafd64afaa8 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchRequestTest.java
@@ -38,47 +38,47 @@ public void testBatchRequest() {
         .delete("b1", "o1")
         .delete("b1", "o2", BlobSourceOption.generationMatch(1),
             BlobSourceOption.metagenerationMatch(2))
-        .update(Blob.of("b2", "o1"), BlobTargetOption.predefinedAcl(PUBLIC_READ))
-        .update(Blob.of("b2", "o2"))
+        .update(BlobInfo.of("b2", "o1"), BlobTargetOption.predefinedAcl(PUBLIC_READ))
+        .update(BlobInfo.of("b2", "o2"))
         .get("b3", "o1")
         .get("b3", "o2", BlobSourceOption.generationMatch(1))
         .get("b3", "o3")
         .build();
 
-    Iterator>> deletes = request
+    Iterator>> deletes = request
         .toDelete().entrySet().iterator();
-    Entry> delete = deletes.next();
-    assertEquals(Blob.of("b1", "o1"), delete.getKey());
+    Entry> delete = deletes.next();
+    assertEquals(BlobInfo.of("b1", "o1"), delete.getKey());
     assertTrue(Iterables.isEmpty(delete.getValue()));
     delete = deletes.next();
-    assertEquals(Blob.of("b1", "o2"), delete.getKey());
+    assertEquals(BlobInfo.of("b1", "o2"), delete.getKey());
     assertEquals(2, Iterables.size(delete.getValue()));
     assertFalse(deletes.hasNext());
 
-    Iterator>> updates = request
+    Iterator>> updates = request
         .toUpdate().entrySet().iterator();
-    Entry> update = updates.next();
-    assertEquals(Blob.of("b2", "o1"), update.getKey());
+    Entry> update = updates.next();
+    assertEquals(BlobInfo.of("b2", "o1"), update.getKey());
     assertEquals(1, Iterables.size(update.getValue()));
     assertEquals(BlobTargetOption.predefinedAcl(PUBLIC_READ),
         Iterables.getFirst(update.getValue(), null));
     update = updates.next();
-    assertEquals(Blob.of("b2", "o2"), update.getKey());
+    assertEquals(BlobInfo.of("b2", "o2"), update.getKey());
     assertTrue(Iterables.isEmpty(update.getValue()));
     assertFalse(updates.hasNext());
 
-    Iterator>> gets = request
+    Iterator>> gets = request
         .toGet().entrySet().iterator();
-    Entry> get = gets.next();
-    assertEquals(Blob.of("b3", "o1"), get.getKey());
+    Entry> get = gets.next();
+    assertEquals(BlobInfo.of("b3", "o1"), get.getKey());
     assertTrue(Iterables.isEmpty(get.getValue()));
     get = gets.next();
-    assertEquals(Blob.of("b3", "o2"), get.getKey());
+    assertEquals(BlobInfo.of("b3", "o2"), get.getKey());
     assertEquals(1, Iterables.size(get.getValue()));
     assertEquals(BlobSourceOption.generationMatch(1),
         Iterables.getFirst(get.getValue(), null));
     get = gets.next();
-    assertEquals(Blob.of("b3", "o3"), get.getKey());
+    assertEquals(BlobInfo.of("b3", "o3"), get.getKey());
     assertTrue(Iterables.isEmpty(get.getValue()));
     assertFalse(gets.hasNext());
   }
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchResponseTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
index 277c46860ef1..7e774a77c739 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BatchResponseTest.java
@@ -27,15 +27,15 @@
 
 public class BatchResponseTest {
 
-  private static final Blob BLOB1 = Blob.of("b", "o1");
-  private static final Blob BLOB2 = Blob.of("b", "o2");
-  private static final Blob BLOB3 = Blob.of("b", "o3");
+  private static final BlobInfo BLOB_INFO_1 = BlobInfo.of("b", "o1");
+  private static final BlobInfo BLOB_INFO_2 = BlobInfo.of("b", "o2");
+  private static final BlobInfo BLOB_INFO_3 = BlobInfo.of("b", "o3");
 
   @Test
   public void testBatchResponse() {
     List> deletes = ImmutableList.of(Result.of(true), Result.of(false));
-    List> updates = ImmutableList.of(Result.of(BLOB1), Result.of(BLOB2));
-    List> gets = ImmutableList.of(Result.of(BLOB2), Result.of(BLOB3));
+    List> updates = ImmutableList.of(Result.of(BLOB_INFO_1), Result.of(BLOB_INFO_2));
+    List> gets = ImmutableList.of(Result.of(BLOB_INFO_2), Result.of(BLOB_INFO_3));
     BatchResponse response = new BatchResponse(deletes, updates, gets);
 
     assertEquals(deletes, response.deletes());
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java
similarity index 70%
rename from gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java
index 6fb8f0243987..018a59c9cfc4 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java
@@ -31,7 +31,7 @@
 import java.util.List;
 import java.util.Map;
 
-public class BlobTest {
+public class BlobInfoTest {
 
   private static final List ACL = ImmutableList.of(
       new Acl(User.ofAllAuthenticatedUsers(), READER),
@@ -55,7 +55,7 @@ public class BlobTest {
   private static final String SELF_LINK = "http://storage/b/n";
   private static final Long SIZE = 1024L;
   private static final Long UPDATE_TIME = DELETE_TIME - 1L;
-  private static final Blob BLOB = Blob.builder("b", "n")
+  private static final BlobInfo BLOB_INFO = BlobInfo.builder("b", "n")
       .acl(ACL)
       .componentCount(COMPONENT_COUNT)
       .contentType(CONTENT_TYPE)
@@ -80,49 +80,49 @@ public class BlobTest {
 
   @Test
   public void testToBuilder() {
-    compareBlobs(BLOB, BLOB.toBuilder().build());
-    Blob blob = BLOB.toBuilder().name("n2").bucket("b2").size(200L).build();
-    assertEquals("n2", blob.name());
-    assertEquals("b2", blob.bucket());
-    assertEquals(Long.valueOf(200), blob.size());
-    blob = blob.toBuilder().name("n").bucket("b").size(SIZE).build();
-    compareBlobs(BLOB, blob);
+    compareBlobs(BLOB_INFO, BLOB_INFO.toBuilder().build());
+    BlobInfo blobInfo = BLOB_INFO.toBuilder().name("n2").bucket("b2").size(200L).build();
+    assertEquals("n2", blobInfo.name());
+    assertEquals("b2", blobInfo.bucket());
+    assertEquals(Long.valueOf(200), blobInfo.size());
+    blobInfo = blobInfo.toBuilder().name("n").bucket("b").size(SIZE).build();
+    compareBlobs(BLOB_INFO, blobInfo);
   }
 
   @Test
   public void testOf() {
-    Blob blob = Blob.of("b", "n");
-    assertEquals("b", blob.bucket());
-    assertEquals("n", blob.name());
+    BlobInfo blobInfo = BlobInfo.of("b", "n");
+    assertEquals("b", blobInfo.bucket());
+    assertEquals("n", blobInfo.name());
   }
 
   @Test
   public void testBuilder() {
-    assertEquals("b", BLOB.bucket());
-    assertEquals("n", BLOB.name());
-    assertEquals(ACL, BLOB.acl());
-    assertEquals(COMPONENT_COUNT, BLOB.componentCount());
-    assertEquals(CONTENT_TYPE, BLOB.contentType());
-    assertEquals(CACHE_CONTROL, BLOB.cacheControl() );
-    assertEquals(CONTENT_DISPOSITION, BLOB.contentDisposition());
-    assertEquals(CONTENT_ENCODING, BLOB.contentEncoding());
-    assertEquals(CONTENT_LANGUAGE, BLOB.contentLanguage());
-    assertEquals(CRC32, BLOB.crc32c());
-    assertEquals(DELETE_TIME, BLOB.deleteTime());
-    assertEquals(ETAG, BLOB.etag());
-    assertEquals(GENERATION, BLOB.generation());
-    assertEquals(ID, BLOB.id());
-    assertEquals(MD5, BLOB.md5());
-    assertEquals(MEDIA_LINK, BLOB.mediaLink());
-    assertEquals(METADATA, BLOB.metadata());
-    assertEquals(META_GENERATION, BLOB.metageneration());
-    assertEquals(OWNER, BLOB.owner());
-    assertEquals(SELF_LINK, BLOB.selfLink());
-    assertEquals(SIZE, BLOB.size());
-    assertEquals(UPDATE_TIME, BLOB.updateTime());
+    assertEquals("b", BLOB_INFO.bucket());
+    assertEquals("n", BLOB_INFO.name());
+    assertEquals(ACL, BLOB_INFO.acl());
+    assertEquals(COMPONENT_COUNT, BLOB_INFO.componentCount());
+    assertEquals(CONTENT_TYPE, BLOB_INFO.contentType());
+    assertEquals(CACHE_CONTROL, BLOB_INFO.cacheControl() );
+    assertEquals(CONTENT_DISPOSITION, BLOB_INFO.contentDisposition());
+    assertEquals(CONTENT_ENCODING, BLOB_INFO.contentEncoding());
+    assertEquals(CONTENT_LANGUAGE, BLOB_INFO.contentLanguage());
+    assertEquals(CRC32, BLOB_INFO.crc32c());
+    assertEquals(DELETE_TIME, BLOB_INFO.deleteTime());
+    assertEquals(ETAG, BLOB_INFO.etag());
+    assertEquals(GENERATION, BLOB_INFO.generation());
+    assertEquals(ID, BLOB_INFO.id());
+    assertEquals(MD5, BLOB_INFO.md5());
+    assertEquals(MEDIA_LINK, BLOB_INFO.mediaLink());
+    assertEquals(METADATA, BLOB_INFO.metadata());
+    assertEquals(META_GENERATION, BLOB_INFO.metageneration());
+    assertEquals(OWNER, BLOB_INFO.owner());
+    assertEquals(SELF_LINK, BLOB_INFO.selfLink());
+    assertEquals(SIZE, BLOB_INFO.size());
+    assertEquals(UPDATE_TIME, BLOB_INFO.updateTime());
   }
 
-  private void compareBlobs(Blob expected, Blob value) {
+  private void compareBlobs(BlobInfo expected, BlobInfo value) {
     assertEquals(expected, value);
     assertEquals(expected.bucket(), value.bucket());
     assertEquals(expected.name(), value.name());
@@ -150,6 +150,6 @@ private void compareBlobs(Blob expected, Blob value) {
 
   @Test
   public void testToPbAndFromPb() {
-    compareBlobs(BLOB, Blob.fromPb(BLOB.toPb()));
+    compareBlobs(BLOB_INFO, BlobInfo.fromPb(BLOB_INFO.toPb()));
   }
 }
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
similarity index 72%
rename from gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java
rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
index c095a6291e49..27a5b4014f3f 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BucketInfoTest.java
@@ -26,22 +26,22 @@
 import com.google.gcloud.storage.Acl.Project;
 import com.google.gcloud.storage.Acl.Role;
 import com.google.gcloud.storage.Acl.User;
-import com.google.gcloud.storage.Bucket.AgeDeleteRule;
-import com.google.gcloud.storage.Bucket.CreatedBeforeDeleteRule;
-import com.google.gcloud.storage.Bucket.DeleteRule;
-import com.google.gcloud.storage.Bucket.DeleteRule.Type;
-import com.google.gcloud.storage.Bucket.IsLiveDeleteRule;
-import com.google.gcloud.storage.Bucket.Location;
-import com.google.gcloud.storage.Bucket.NumNewerVersionsDeleteRule;
-import com.google.gcloud.storage.Bucket.RawDeleteRule;
-import com.google.gcloud.storage.Bucket.StorageClass;
+import com.google.gcloud.storage.BucketInfo.AgeDeleteRule;
+import com.google.gcloud.storage.BucketInfo.CreatedBeforeDeleteRule;
+import com.google.gcloud.storage.BucketInfo.DeleteRule;
+import com.google.gcloud.storage.BucketInfo.DeleteRule.Type;
+import com.google.gcloud.storage.BucketInfo.IsLiveDeleteRule;
+import com.google.gcloud.storage.BucketInfo.Location;
+import com.google.gcloud.storage.BucketInfo.NumNewerVersionsDeleteRule;
+import com.google.gcloud.storage.BucketInfo.RawDeleteRule;
+import com.google.gcloud.storage.BucketInfo.StorageClass;
 
 import org.junit.Test;
 
 import java.util.Collections;
 import java.util.List;
 
-public class BucketTest {
+public class BucketInfoTest {
 
   private static final List ACL = ImmutableList.of(
       new Acl(User.ofAllAuthenticatedUsers(), Role.READER),
@@ -62,7 +62,7 @@ public class BucketTest {
   private static final Location LOCATION = Location.asia();
   private static final StorageClass STORAGE_CLASS = StorageClass.standard();
   private static final Boolean VERSIONING_ENABLED = true;
-  private static final Bucket BUCKET = Bucket.builder("b")
+  private static final BucketInfo BUCKET_INFO = BucketInfo.builder("b")
       .acl(ACL)
       .etag(ETAG)
       .id(ID)
@@ -82,46 +82,46 @@ public class BucketTest {
 
   @Test
   public void testToBuilder() {
-    compareBuckets(BUCKET, BUCKET.toBuilder().build());
-    Bucket bucket = BUCKET.toBuilder().name("B").id("id").build();
-    assertEquals("B", bucket.name());
-    assertEquals("id", bucket.id());
-    bucket = bucket.toBuilder().name("b").id(ID).build();
-    compareBuckets(BUCKET, bucket);
+    compareBuckets(BUCKET_INFO, BUCKET_INFO.toBuilder().build());
+    BucketInfo bucketInfo = BUCKET_INFO.toBuilder().name("B").id("id").build();
+    assertEquals("B", bucketInfo.name());
+    assertEquals("id", bucketInfo.id());
+    bucketInfo = bucketInfo.toBuilder().name("b").id(ID).build();
+    compareBuckets(BUCKET_INFO, bucketInfo);
   }
 
   @Test
   public void testOf() {
-    Bucket bucket = Bucket.of("bucket");
-    assertEquals("bucket", bucket.name());
+    BucketInfo bucketInfo = BucketInfo.of("bucket");
+    assertEquals("bucket", bucketInfo.name());
   }
 
   @Test
   public void testBuilder() {
-    assertEquals("b", BUCKET.name());
-    assertEquals(ACL, BUCKET.acl());
-    assertEquals(ETAG, BUCKET.etag());
-    assertEquals(ID, BUCKET.id());
-    assertEquals(META_GENERATION, BUCKET.metageneration());
-    assertEquals(OWNER, BUCKET.owner());
-    assertEquals(SELF_LINK, BUCKET.selfLink());
-    assertEquals(CREATE_TIME, BUCKET.createTime());
-    assertEquals(CORS, BUCKET.cors());
-    assertEquals(DEFAULT_ACL, BUCKET.defaultAcl());
-    assertEquals(DELETE_RULES, BUCKET.deleteRules());
-    assertEquals(INDEX_PAGE, BUCKET.indexPage());
-    assertEquals(NOT_FOUND_PAGE, BUCKET.notFoundPage());
-    assertEquals(LOCATION, BUCKET.location());
-    assertEquals(STORAGE_CLASS, BUCKET.storageClass());
-    assertEquals(VERSIONING_ENABLED, BUCKET.versioningEnabled());
+    assertEquals("b", BUCKET_INFO.name());
+    assertEquals(ACL, BUCKET_INFO.acl());
+    assertEquals(ETAG, BUCKET_INFO.etag());
+    assertEquals(ID, BUCKET_INFO.id());
+    assertEquals(META_GENERATION, BUCKET_INFO.metageneration());
+    assertEquals(OWNER, BUCKET_INFO.owner());
+    assertEquals(SELF_LINK, BUCKET_INFO.selfLink());
+    assertEquals(CREATE_TIME, BUCKET_INFO.createTime());
+    assertEquals(CORS, BUCKET_INFO.cors());
+    assertEquals(DEFAULT_ACL, BUCKET_INFO.defaultAcl());
+    assertEquals(DELETE_RULES, BUCKET_INFO.deleteRules());
+    assertEquals(INDEX_PAGE, BUCKET_INFO.indexPage());
+    assertEquals(NOT_FOUND_PAGE, BUCKET_INFO.notFoundPage());
+    assertEquals(LOCATION, BUCKET_INFO.location());
+    assertEquals(STORAGE_CLASS, BUCKET_INFO.storageClass());
+    assertEquals(VERSIONING_ENABLED, BUCKET_INFO.versioningEnabled());
   }
 
   @Test
   public void testToPbAndFromPb() {
-    compareBuckets(BUCKET, Bucket.fromPb(BUCKET.toPb()));
+    compareBuckets(BUCKET_INFO, BucketInfo.fromPb(BUCKET_INFO.toPb()));
   }
 
-  private void compareBuckets(Bucket expected, Bucket value) {
+  private void compareBuckets(BucketInfo expected, BucketInfo value) {
     assertEquals(expected, value);
     assertEquals(expected.name(), value.name());
     assertEquals(expected.acl(), value.acl());
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
index d765b0189c96..e79163da3a7e 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -40,18 +40,18 @@ public class SerializationTest {
   private static final Acl.Project ACL_PROJECT_ = new Acl.Project(ProjectRole.VIEWERS, "pid");
   private static final Acl.User ACL_USER = new Acl.User("user");
   private static final Acl.RawEntity ACL_RAW = new Acl.RawEntity("raw");
-  private static final Blob BLOB = Blob.of("b", "n");
-  private static final Bucket BUCKET = Bucket.of("b");
+  private static final BlobInfo BLOB_INFO = BlobInfo.of("b", "n");
+  private static final BucketInfo BUCKET_INFO = BucketInfo.of("b");
   private static final Cors.Origin ORIGIN = Cors.Origin.any();
   private static final Cors CORS =
       Cors.builder().maxAgeSeconds(1).origins(Collections.singleton(ORIGIN)).build();
   private static final BatchRequest BATCH_REQUEST = BatchRequest.builder().delete("B", "N").build();
   private static final BatchResponse BATCH_RESPONSE = new BatchResponse(
       Collections.singletonList(BatchResponse.Result.of(true)),
-      Collections.>emptyList(),
-      Collections.>emptyList());
-  private static final ListResult LIST_RESULT =
-      new ListResult<>(null, "c", Collections.singletonList(Blob.of("b", "n")));
+      Collections.>emptyList(),
+      Collections.>emptyList());
+  private static final ListResult LIST_RESULT =
+      new ListResult<>(null, "c", Collections.singletonList(BlobInfo.of("b", "n")));
   private static Storage.BlobListOption BLOB_LIST_OPTIONS =
       Storage.BlobListOption.maxResults(100);
   private static Storage.BlobSourceOption BLOB_SOURCE_OPTIONS =
@@ -86,7 +86,8 @@ public void testServiceOptions() throws Exception {
 
   @Test
   public void testModelAndRequests() throws Exception {
-    Serializable[] objects = {ACL_DOMAIN, ACL_GROUP, ACL_PROJECT_, ACL_USER, ACL_RAW, BLOB, BUCKET,
+    Serializable[] objects = {ACL_DOMAIN, ACL_GROUP, ACL_PROJECT_, ACL_USER, ACL_RAW, BLOB_INFO,
+        BUCKET_INFO,
       ORIGIN, CORS, BATCH_REQUEST,BATCH_RESPONSE, LIST_RESULT, BLOB_LIST_OPTIONS,
         BLOB_SOURCE_OPTIONS, BLOB_TARGET_OPTIONS, BUCKET_LIST_OPTIONS, BUCKET_SOURCE_OPTIONS,
         BUCKET_TARGET_OPTIONS};

From c47b9db43f90c346a600372ac5985820728cce52 Mon Sep 17 00:00:00 2001
From: ozarov 
Date: Tue, 2 Jun 2015 21:20:08 -0700
Subject: [PATCH 256/732] rename load to readAllBytes

---
 .../main/java/com/google/gcloud/examples/StorageExample.java  | 2 +-
 .../src/main/java/com/google/gcloud/storage/Storage.java      | 4 ++--
 .../src/main/java/com/google/gcloud/storage/StorageImpl.java  | 2 +-
 .../src/main/java/com/google/gcloud/storage/package-info.java | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
index 76016a1c8790..26437b995bd2 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java
@@ -329,7 +329,7 @@ private void run(Storage storage, String bucket, String blobName, Path downloadT
       }
       if (blobInfo.size() < 1_000_000) {
         // Blob is small read all its content in one request
-        byte[] content = storage.load(blobInfo.bucket(), blobInfo.name());
+        byte[] content = storage.readAllBytes(blobInfo.bucket(), blobInfo.name());
         writeTo.write(content);
       } else {
         // When Blob size is big or unknown use the blob's channel reader.
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index 770a5a924ff6..6427bfaccf58 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -560,12 +560,12 @@ public static Builder builder() {
   BlobInfo copy(CopyRequest copyRequest);
 
   /**
-   * Load the content of the given blob.
+   * Reads all the bytes from a blob.
    *
    * @return the blob's content.
    * @throws StorageException upon failure
    */
-  byte[] load(String bucket, String blob, BlobSourceOption... options);
+  byte[] readAllBytes(String bucket, String blob, BlobSourceOption... options);
 
   /**
    * Send a batch request.
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index 44c940f1c64a..6eb8e3359a4d 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -354,7 +354,7 @@ public StorageObject call() {
   }
 
   @Override
-  public byte[] load(String bucket, String blob, BlobSourceOption... options) {
+  public byte[] readAllBytes(String bucket, String blob, BlobSourceOption... options) {
     final StorageObject storageObject = BlobInfo.of(bucket, blob).toPb();
     final Map optionsMap = optionMap(options);
     return runWithRetries(new Callable() {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
index f689d264a828..cb43dd0be1b2 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/package-info.java
@@ -26,7 +26,7 @@
  * if (blobInfo == null) {
  *   storage.create(BlobInfo.of("bucket", "blob_name"), content);
  * } else {
- *   byte[] prevContent = storage.load("bucket", "blob_name");
+ *   byte[] prevContent = storage.readAllBytes("bucket", "blob_name");
  *   content = mergeContent(prevContent, content);
  *   WritableByteChannel channel = storage.writer(blob);
  *   channel.write(ByteBuffer.wrap(content));

From 04aef423549873d3ca265ad8baa977216c81675d Mon Sep 17 00:00:00 2001
From: Jeremy Boynes 
Date: Wed, 3 Jun 2015 09:55:52 -0700
Subject: [PATCH 257/732] Replace CR characters with newlines

---
 .../java/com/google/gcloud/BaseService.java   |  32 +-
 .../main/java/com/google/gcloud/Service.java  |  22 +-
 .../google/gcloud/spi/ServiceRpcFactory.java  |  31 +-
 .../com/google/gcloud/spi/DatastoreRpc.java   | 118 ++++++-
 .../BaseDatastoreBatchWriterTest.java         | 321 +++++++++++++++++-
 .../gcloud/datastore/BaseEntityTest.java      | 207 ++++++++++-
 .../com/google/gcloud/datastore/BlobTest.java | 112 +++++-
 .../google/gcloud/datastore/CursorTest.java   |  53 ++-
 .../datastore/DatastoreExceptionTest.java     |  66 +++-
 .../google/gcloud/datastore/DateTimeTest.java |  78 ++++-
 .../com/google/gcloud/spi/StorageRpc.java     | 172 +++++++++-
 .../gcloud/storage/StorageException.java      |  49 ++-
 12 files changed, 1249 insertions(+), 12 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java b/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
index 4d96ac5b86e6..02bfd76771b2 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
@@ -1 +1,31 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud;

public abstract class BaseService implements Service {

  private final O options;

  protected BaseService(O options) {
    this.options = options;
  }

  @Override
  public O options() {
    return options;
  }
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud;
+
+public abstract class BaseService implements Service {
+
+  private final O options;
+
+  protected BaseService(O options) {
+    this.options = options;
+  }
+
+  @Override
+  public O options() {
+    return options;
+  }
+}
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/Service.java b/gcloud-java-core/src/main/java/com/google/gcloud/Service.java
index 02d810186bfc..8a43b9cf2244 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/Service.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/Service.java
@@ -1 +1,21 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud;

public interface Service {
  O options();
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud;
+
+public interface Service {
+  O options();
+}
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
index e8e67615305d..9bab3f81ce9c 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -1 +1,30 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.gcloud.ServiceOptions;

import java.io.Serializable;

/**
 * A base interface for all service RPC factories.
 * Loading of a factory implementation is done via {@link java.util.ServiceLoader}.
 */
public interface ServiceRpcFactory extends Serializable {

  S create(O options);
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.spi;
+
+import com.google.gcloud.ServiceOptions;
+
+import java.io.Serializable;
+
+/**
+ * A base interface for all service RPC factories.
+ * Loading of a factory implementation is done via {@link java.util.ServiceLoader}.
+ */
+public interface ServiceRpcFactory extends Serializable {
+
+  S create(O options);
+}
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
index be194c7b3848..f4b7cd39587b 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
@@ -1 +1,117 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.gcloud.spi;

import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
import com.google.api.services.datastore.DatastoreV1.CommitRequest;
import com.google.api.services.datastore.DatastoreV1.CommitResponse;
import com.google.api.services.datastore.DatastoreV1.LookupRequest;
import com.google.api.services.datastore.DatastoreV1.LookupResponse;
import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;

/**
 * Provides access to the remote Datastore service.
 */
public interface DatastoreRpc {

  public class DatastoreRpcException extends Exception {

    /**
     * The reason for the exception.
     *
     * @see Google Cloud Datastore error codes
     */
    public enum Reason {

      ABORTED(true, "Request aborted", 409),
      DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
      FAILED_PRECONDITION(false, "Invalid request", 412),
      INTERNAL(false, "Server returned an error", 500),
      INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
      PERMISSION_DENIED(false, "Unauthorized request", 403),
      RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
      UNAVAILABLE(true, "Could not reach service", 503);

      private final boolean retryable;
      private final String description;
      private final int httpStatus;

      private Reason(boolean retryable, String description, int httpStatus) {
        this.retryable = retryable;
        this.description = description;
        this.httpStatus = httpStatus;
      }

      public boolean retryable() {
        return retryable;
      }

      public String description() {
        return description;
      }

      public int httpStatus() {
        return httpStatus;
      }
    }

    private final String reason;
    private final int httpStatus;
    private final boolean retryable;

    public DatastoreRpcException(Reason reason) {
      this(reason.name(), reason.httpStatus, reason.retryable, reason.description);
    }

    public DatastoreRpcException(String reason, int httpStatus, boolean retryable, String message) {
      super(message);
      this.reason = reason;
      this.httpStatus = httpStatus;
      this.retryable = retryable;
    }

    public String reason() {
      return reason;
    }

    public int httpStatus() {
      return httpStatus;
    }

    public boolean retryable() {
      return retryable;
    }
  }

  AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException;

  BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
      throws DatastoreRpcException;

  CommitResponse commit(CommitRequest request) throws DatastoreRpcException;

  LookupResponse lookup(LookupRequest request) throws DatastoreRpcException;

  RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException;

  RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException;
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.gcloud.spi;
+
+import com.google.api.services.datastore.DatastoreV1.AllocateIdsRequest;
+import com.google.api.services.datastore.DatastoreV1.AllocateIdsResponse;
+import com.google.api.services.datastore.DatastoreV1.BeginTransactionRequest;
+import com.google.api.services.datastore.DatastoreV1.BeginTransactionResponse;
+import com.google.api.services.datastore.DatastoreV1.CommitRequest;
+import com.google.api.services.datastore.DatastoreV1.CommitResponse;
+import com.google.api.services.datastore.DatastoreV1.LookupRequest;
+import com.google.api.services.datastore.DatastoreV1.LookupResponse;
+import com.google.api.services.datastore.DatastoreV1.RollbackRequest;
+import com.google.api.services.datastore.DatastoreV1.RollbackResponse;
+import com.google.api.services.datastore.DatastoreV1.RunQueryRequest;
+import com.google.api.services.datastore.DatastoreV1.RunQueryResponse;
+
+/**
+ * Provides access to the remote Datastore service.
+ */
+public interface DatastoreRpc {
+
+  public class DatastoreRpcException extends Exception {
+
+    /**
+     * The reason for the exception.
+     *
+     * @see Google Cloud Datastore error codes
+     */
+    public enum Reason {
+
+      ABORTED(true, "Request aborted", 409),
+      DEADLINE_EXCEEDED(true, "Deadline exceeded", 403),
+      FAILED_PRECONDITION(false, "Invalid request", 412),
+      INTERNAL(false, "Server returned an error", 500),
+      INVALID_ARGUMENT(false, "Request parameter has an invalid value", 400),
+      PERMISSION_DENIED(false, "Unauthorized request", 403),
+      RESOURCE_EXHAUSTED(false, "Quota exceeded", 402),
+      UNAVAILABLE(true, "Could not reach service", 503);
+
+      private final boolean retryable;
+      private final String description;
+      private final int httpStatus;
+
+      private Reason(boolean retryable, String description, int httpStatus) {
+        this.retryable = retryable;
+        this.description = description;
+        this.httpStatus = httpStatus;
+      }
+
+      public boolean retryable() {
+        return retryable;
+      }
+
+      public String description() {
+        return description;
+      }
+
+      public int httpStatus() {
+        return httpStatus;
+      }
+    }
+
+    private final String reason;
+    private final int httpStatus;
+    private final boolean retryable;
+
+    public DatastoreRpcException(Reason reason) {
+      this(reason.name(), reason.httpStatus, reason.retryable, reason.description);
+    }
+
+    public DatastoreRpcException(String reason, int httpStatus, boolean retryable, String message) {
+      super(message);
+      this.reason = reason;
+      this.httpStatus = httpStatus;
+      this.retryable = retryable;
+    }
+
+    public String reason() {
+      return reason;
+    }
+
+    public int httpStatus() {
+      return httpStatus;
+    }
+
+    public boolean retryable() {
+      return retryable;
+    }
+  }
+
+  AllocateIdsResponse allocateIds(AllocateIdsRequest request) throws DatastoreRpcException;
+
+  BeginTransactionResponse beginTransaction(BeginTransactionRequest request)
+      throws DatastoreRpcException;
+
+  CommitResponse commit(CommitRequest request) throws DatastoreRpcException;
+
+  LookupResponse lookup(LookupRequest request) throws DatastoreRpcException;
+
+  RollbackResponse rollback(RollbackRequest request) throws DatastoreRpcException;
+
+  RunQueryResponse runQuery(RunQueryRequest request) throws DatastoreRpcException;
+}
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
index c3f1bfbd5a71..9fe207a04056 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
@@ -1 +1,320 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.easymock.EasyMock.*;
import static org.junit.Assert.assertEquals;

import com.google.api.services.datastore.DatastoreV1;
import com.google.common.collect.ImmutableList;
import org.easymock.EasyMock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.util.List;

public class BaseDatastoreBatchWriterTest {

  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
  private static final Key KEY2 = Key.builder(KEY1, 1).build();
  private static final Key KEY3 = Key.builder(KEY1, 2).build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder(KEY1).build();
  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
  private static final FullEntity INCOMPLETE_ENTITY_1 =
      Entity.builder(INCOMPLETE_KEY).build();
  private static final FullEntity INCOMPLETE_ENTITY_2 =
      Entity.builder(INCOMPLETE_KEY).set("name", "dan").build();

  private DatastoreBatchWriter batchWriter;

  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {

    private final Datastore datastore;

    protected DatastoreBatchWriter() {
      super("test");
      datastore = EasyMock.createMock(Datastore.class);
      IncompleteKey[] expected = {INCOMPLETE_KEY, INCOMPLETE_KEY};
      List result = ImmutableList.of(KEY2, KEY3);
      expect(datastore.allocateId(expected)).andReturn(result).times(0, 1);
      replay(datastore);
    }

    @Override
    protected Datastore datastore() {
      return datastore;
    }

    void finish() {
      verify(datastore);
    }
  }

  @Before
  public void setUp() {
    batchWriter = new DatastoreBatchWriter();
  }

  @After
  public void tearDown() {
    batchWriter.finish();
  }

  @Test
  public void testAdd() throws Exception {
    Entity entity2 =
        Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsert(entity2.toPb())
        .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb())
        .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb())
        .build();
    List entities = batchWriter
        .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2);
    assertEquals(pb, batchWriter.toMutationPb().build());
    assertEquals(ENTITY1, entities.get(0));
    assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1));
    assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2));
    assertEquals(entity2, entities.get(3));
  }

  @Test
  public void testAddAfterDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.add(ENTITY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testAddDuplicate() throws Exception {
    batchWriter.add(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreException.class)
  public void testAddAfterPut() throws Exception {
    batchWriter.put(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreException.class)
  public void testAddAfterUpdate() throws Exception {
    batchWriter.update(ENTITY1);
    batchWriter.add(ENTITY1);
  }

  @Test(expected = DatastoreException.class)
  public void testAddWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.add(ENTITY1);
  }

  @Test
  public void testAddWithDeferredAllocation() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsert(ENTITY1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb())
        .build();
    batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testAddWithDeferredAllocationWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
  }

  @Test
  public void testUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(ENTITY1.toPb())
        .addUpdate(ENTITY2.toPb())
        .addUpdate(ENTITY3.toPb())
        .build();
    batchWriter.update(ENTITY1, ENTITY2);
    batchWriter.update(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpdate(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testUpdateAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.update(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testUpdateAfterDelete() throws Exception {
    batchWriter.delete(KEY1);
    batchWriter.update(ENTITY1, ENTITY2);
  }

  @Test(expected = DatastoreException.class)
  public void testUpdateWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.update(ENTITY1);
  }

  @Test
  public void testPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(ENTITY1.toPb())
        .addUpsert(ENTITY2.toPb())
        .addUpsert(ENTITY3.toPb())
        .build();
    batchWriter.put(ENTITY1, ENTITY2);
    batchWriter.put(ENTITY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterPut() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterAdd() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterUpdate() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testPutAfterDelete() throws Exception {
    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addUpsert(entity.toPb())
        .build();
    batchWriter.delete(KEY1);
    batchWriter.put(entity);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testPutWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.put(ENTITY1);
  }

  @Test
  public void testDelete() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .addDelete(KEY2.toPb())
        .addDelete(KEY3.toPb())
        .build();
    batchWriter.delete(KEY1, KEY2);
    batchWriter.delete(KEY3);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterAdd() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.add(ENTITY1);
    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterUpdate() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.update(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test
  public void testDeleteAfterPut() throws Exception {
    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
        .addDelete(KEY1.toPb())
        .build();
    batchWriter.put(ENTITY1);
    batchWriter.delete(KEY1);
    assertEquals(pb, batchWriter.toMutationPb().build());
  }

  @Test(expected = DatastoreException.class)
  public void testDeleteWhenNotActive() throws Exception {
    batchWriter.deactivate();
    batchWriter.delete(KEY1);
  }
}

\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+
+import com.google.api.services.datastore.DatastoreV1;
+import com.google.common.collect.ImmutableList;
+import org.easymock.EasyMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+public class BaseDatastoreBatchWriterTest {
+
+  private static final Key KEY1 = Key.builder("dataset1", "kind1", "name1").build();
+  private static final Key KEY2 = Key.builder(KEY1, 1).build();
+  private static final Key KEY3 = Key.builder(KEY1, 2).build();
+  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder(KEY1).build();
+  private static final Entity ENTITY1 = Entity.builder(KEY1).build();
+  private static final Entity ENTITY2 = Entity.builder(KEY2).set("bak", true).build();
+  private static final Entity ENTITY3 = Entity.builder(KEY3).set("bak", true).build();
+  private static final FullEntity INCOMPLETE_ENTITY_1 =
+      Entity.builder(INCOMPLETE_KEY).build();
+  private static final FullEntity INCOMPLETE_ENTITY_2 =
+      Entity.builder(INCOMPLETE_KEY).set("name", "dan").build();
+
+  private DatastoreBatchWriter batchWriter;
+
+  private class DatastoreBatchWriter extends BaseDatastoreBatchWriter {
+
+    private final Datastore datastore;
+
+    protected DatastoreBatchWriter() {
+      super("test");
+      datastore = EasyMock.createMock(Datastore.class);
+      IncompleteKey[] expected = {INCOMPLETE_KEY, INCOMPLETE_KEY};
+      List result = ImmutableList.of(KEY2, KEY3);
+      expect(datastore.allocateId(expected)).andReturn(result).times(0, 1);
+      replay(datastore);
+    }
+
+    @Override
+    protected Datastore datastore() {
+      return datastore;
+    }
+
+    void finish() {
+      verify(datastore);
+    }
+  }
+
+  @Before
+  public void setUp() {
+    batchWriter = new DatastoreBatchWriter();
+  }
+
+  @After
+  public void tearDown() {
+    batchWriter.finish();
+  }
+
+  @Test
+  public void testAdd() throws Exception {
+    Entity entity2 =
+        Entity.builder(ENTITY2).key(Key.builder(KEY1).name("name2").build()).build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addInsert(ENTITY1.toPb())
+        .addInsert(entity2.toPb())
+        .addInsert(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build().toPb())
+        .addInsert(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build().toPb())
+        .build();
+    List entities = batchWriter
+        .add(ENTITY1, INCOMPLETE_ENTITY_1, INCOMPLETE_ENTITY_2, entity2);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+    assertEquals(ENTITY1, entities.get(0));
+    assertEquals(Entity.builder(KEY2, INCOMPLETE_ENTITY_1).build(), entities.get(1));
+    assertEquals(Entity.builder(KEY3, INCOMPLETE_ENTITY_2).build(), entities.get(2));
+    assertEquals(entity2, entities.get(3));
+  }
+
+  @Test
+  public void testAddAfterDelete() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(ENTITY1.toPb())
+        .build();
+    batchWriter.delete(KEY1);
+    batchWriter.add(ENTITY1);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testAddDuplicate() throws Exception {
+    batchWriter.add(ENTITY1);
+    batchWriter.add(ENTITY1);
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testAddAfterPut() throws Exception {
+    batchWriter.put(ENTITY1);
+    batchWriter.add(ENTITY1);
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testAddAfterUpdate() throws Exception {
+    batchWriter.update(ENTITY1);
+    batchWriter.add(ENTITY1);
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testAddWhenNotActive() throws Exception {
+    batchWriter.deactivate();
+    batchWriter.add(ENTITY1);
+  }
+
+  @Test
+  public void testAddWithDeferredAllocation() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addInsert(ENTITY1.toPb())
+        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
+        .addInsertAutoId(INCOMPLETE_ENTITY_2.toPb())
+        .build();
+    batchWriter.addWithDeferredIdAllocation(ENTITY1, INCOMPLETE_ENTITY_1);
+    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_2);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testAddWithDeferredAllocationWhenNotActive() throws Exception {
+    batchWriter.deactivate();
+    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
+  }
+
+  @Test
+  public void testUpdate() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpdate(ENTITY1.toPb())
+        .addUpdate(ENTITY2.toPb())
+        .addUpdate(ENTITY3.toPb())
+        .build();
+    batchWriter.update(ENTITY1, ENTITY2);
+    batchWriter.update(ENTITY3);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testUpdateAfterUpdate() throws Exception {
+    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpdate(entity.toPb())
+        .build();
+    batchWriter.update(ENTITY1);
+    batchWriter.update(entity);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testUpdateAfterAdd() throws Exception {
+    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(entity.toPb())
+        .build();
+    batchWriter.add(ENTITY1);
+    batchWriter.update(entity);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testUpdateAfterPut() throws Exception {
+    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(entity.toPb())
+        .build();
+    batchWriter.put(ENTITY1);
+    batchWriter.update(entity);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testUpdateAfterDelete() throws Exception {
+    batchWriter.delete(KEY1);
+    batchWriter.update(ENTITY1, ENTITY2);
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testUpdateWhenNotActive() throws Exception {
+    batchWriter.deactivate();
+    batchWriter.update(ENTITY1);
+  }
+
+  @Test
+  public void testPut() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(ENTITY1.toPb())
+        .addUpsert(ENTITY2.toPb())
+        .addUpsert(ENTITY3.toPb())
+        .build();
+    batchWriter.put(ENTITY1, ENTITY2);
+    batchWriter.put(ENTITY3);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testPutAfterPut() throws Exception {
+    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(entity.toPb())
+        .build();
+    batchWriter.put(ENTITY1);
+    batchWriter.put(entity);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testPutAfterAdd() throws Exception {
+    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(entity.toPb())
+        .build();
+    batchWriter.add(ENTITY1);
+    batchWriter.put(entity);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testPutAfterUpdate() throws Exception {
+    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(entity.toPb())
+        .build();
+    batchWriter.update(ENTITY1);
+    batchWriter.put(entity);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testPutAfterDelete() throws Exception {
+    Entity entity = Entity.builder(ENTITY1).set("foo", "bar").build();
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addUpsert(entity.toPb())
+        .build();
+    batchWriter.delete(KEY1);
+    batchWriter.put(entity);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testPutWhenNotActive() throws Exception {
+    batchWriter.deactivate();
+    batchWriter.put(ENTITY1);
+  }
+
+  @Test
+  public void testDelete() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addDelete(KEY1.toPb())
+        .addDelete(KEY2.toPb())
+        .addDelete(KEY3.toPb())
+        .build();
+    batchWriter.delete(KEY1, KEY2);
+    batchWriter.delete(KEY3);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testDeleteAfterAdd() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addInsertAutoId(INCOMPLETE_ENTITY_1.toPb())
+        .addDelete(KEY1.toPb())
+        .build();
+    batchWriter.add(ENTITY1);
+    batchWriter.addWithDeferredIdAllocation(INCOMPLETE_ENTITY_1);
+    batchWriter.delete(KEY1);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testDeleteAfterUpdate() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addDelete(KEY1.toPb())
+        .build();
+    batchWriter.update(ENTITY1);
+    batchWriter.delete(KEY1);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test
+  public void testDeleteAfterPut() throws Exception {
+    DatastoreV1.Mutation pb = DatastoreV1.Mutation.newBuilder()
+        .addDelete(KEY1.toPb())
+        .build();
+    batchWriter.put(ENTITY1);
+    batchWriter.delete(KEY1);
+    assertEquals(pb, batchWriter.toMutationPb().build());
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testDeleteWhenNotActive() throws Exception {
+    batchWriter.deactivate();
+    batchWriter.delete(KEY1);
+  }
+}
+
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
index daa0c502d4b5..02ccd996dc83 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -1 +1,206 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;

import org.junit.Before;
import org.junit.Test;

import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Set;

public class BaseEntityTest {

  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
  private static final DateTime DATE_TIME = DateTime.now();
  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
  private static final FullEntity PARTIAL_ENTITY =
      Entity.builder(INCOMPLETE_KEY).build();

  private Builder builder;

  private class Builder extends BaseEntity.Builder {

    @Override public BaseEntity build() {

      return new BaseEntity(this) {

        @Override
        protected Builder emptyBuilder() {
          return new BaseEntityTest.Builder();
        }
      };
    }
  }

  @Before
  public void setUp() {
    builder = new Builder();
    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
    builder.set("long", 125).setNull("null").set("entity", ENTITY);
    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
    builder.set("list1", NullValue.of(), StringValue.of("foo"));
    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
  }

  @Test
  public void testContains() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.contains("list1"));
    assertFalse(entity.contains("bla"));
    entity = builder.clear().build();
    assertFalse(entity.contains("list1"));
  }

  @Test
  public void testGetValue() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
  }

  @Test(expected = DatastoreException.class)
  public void testGetValueNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.getValue("blob");
  }

  @Test
  public void testIsNull() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.isNull("null"));
    assertFalse(entity.isNull("blob"));
    entity = builder.setNull("blob").build();
    assertTrue(entity.isNull("blob"));
  }

  @Test(expected = DatastoreException.class)
  public void testIsNullNotFound() throws Exception {
    BaseEntity entity = builder.clear().build();
    entity.isNull("null");
  }

  @Test
  public void testGetString() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals("hello world", entity.getString("string"));
    assertEquals("bla", entity.getString("stringValue"));
    entity = builder.set("string", "foo").build();
    assertEquals("foo", entity.getString("string"));
  }

  @Test
  public void testGetLong() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(125, entity.getLong("long"));
    entity = builder.set("long", LongValue.of(10)).build();
    assertEquals(10, entity.getLong("long"));
  }

  @Test
  public void testGetDouble() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(1.25, entity.getDouble("double"), 0);
    entity = builder.set("double", DoubleValue.of(10)).build();
    assertEquals(10, entity.getDouble("double"), 0);
  }

  @Test
  public void testGetBoolean() throws Exception {
    BaseEntity entity = builder.build();
    assertTrue(entity.getBoolean("boolean"));
    entity = builder.set("boolean", BooleanValue.of(false)).build();
    assertFalse(entity.getBoolean("boolean"));
  }

  @Test
  public void testGetDateTime() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
    Calendar cal = Calendar.getInstance();
    cal.add(Calendar.DATE, -1);
    DateTime dateTime = DateTime.copyFrom(cal);
    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
    assertEquals(dateTime, entity.getDateTime("dateTime"));
  }

  @Test
  public void testGetKey() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(KEY, entity.getKey("key"));
    Key key = Key.builder(KEY).name("BLA").build();
    entity = builder.set("key", key).build();
    assertEquals(key, entity.getKey("key"));
  }

  @Test
  public void testGetEntity() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(ENTITY, entity.getEntity("entity"));
    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
  }

  @Test
  public void testGetList() throws Exception {
    BaseEntity entity = builder.build();
    List> list = entity.getList("list1");
    assertEquals(2, list.size());
    assertEquals(NullValue.of(), list.get(0));
    assertEquals("foo", list.get(1).get());
    list = entity.getList("list2");
    assertEquals(2, list.size());
    assertEquals(Long.valueOf(10), list.get(0).get());
    assertEquals(Double.valueOf(2), list.get(1).get());
    list = entity.getList("list3");
    assertEquals(1, list.size());
    assertEquals(Boolean.TRUE, list.get(0).get());
    entity = builder.set("list1", ListValue.of(list)).build();
    assertEquals(list, entity.getList("list1"));
  }

  @Test
  public void testGetBlob() throws Exception {
    BaseEntity entity = builder.build();
    assertEquals(BLOB, entity.getBlob("blob"));
    Blob blob = Blob.copyFrom(new byte[] {});
    entity = builder.set("blob", BlobValue.of(blob)).build();
    assertEquals(blob, entity.getBlob("blob"));
  }

  @Test
  public void testNames() throws Exception {
    Set names = ImmutableSet.builder()
        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
        .build();
    BaseEntity entity = builder.build();
    assertEquals(names, entity.names());
  }
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+public class BaseEntityTest {
+
+  private static final Blob BLOB = Blob.copyFrom(new byte[]{1, 2});
+  private static final DateTime DATE_TIME = DateTime.now();
+  private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
+  private static final Entity ENTITY = Entity.builder(KEY).set("name", "foo").build();
+  private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.builder("ds1", "k1").build();
+  private static final FullEntity PARTIAL_ENTITY =
+      Entity.builder(INCOMPLETE_KEY).build();
+
+  private Builder builder;
+
+  private class Builder extends BaseEntity.Builder {
+
+    @Override public BaseEntity build() {
+
+      return new BaseEntity(this) {
+
+        @Override
+        protected Builder emptyBuilder() {
+          return new BaseEntityTest.Builder();
+        }
+      };
+    }
+  }
+
+  @Before
+  public void setUp() {
+    builder = new Builder();
+    builder.set("blob", BLOB).set("boolean", true).set("dateTime", DATE_TIME);
+    builder.set("double", 1.25).set("key", KEY).set("string", "hello world");
+    builder.set("long", 125).setNull("null").set("entity", ENTITY);
+    builder.set("partialEntity", PARTIAL_ENTITY).set("stringValue", StringValue.of("bla"));
+    builder.set("list1", NullValue.of(), StringValue.of("foo"));
+    builder.set("list2", ImmutableList.of(LongValue.of(10), DoubleValue.of(2)));
+    builder.set("list3", Collections.singletonList(BooleanValue.of(true)));
+  }
+
+  @Test
+  public void testContains() throws Exception {
+    BaseEntity entity = builder.build();
+    assertTrue(entity.contains("list1"));
+    assertFalse(entity.contains("bla"));
+    entity = builder.clear().build();
+    assertFalse(entity.contains("list1"));
+  }
+
+  @Test
+  public void testGetValue() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testGetValueNotFound() throws Exception {
+    BaseEntity entity = builder.clear().build();
+    entity.getValue("blob");
+  }
+
+  @Test
+  public void testIsNull() throws Exception {
+    BaseEntity entity = builder.build();
+    assertTrue(entity.isNull("null"));
+    assertFalse(entity.isNull("blob"));
+    entity = builder.setNull("blob").build();
+    assertTrue(entity.isNull("blob"));
+  }
+
+  @Test(expected = DatastoreException.class)
+  public void testIsNullNotFound() throws Exception {
+    BaseEntity entity = builder.clear().build();
+    entity.isNull("null");
+  }
+
+  @Test
+  public void testGetString() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals("hello world", entity.getString("string"));
+    assertEquals("bla", entity.getString("stringValue"));
+    entity = builder.set("string", "foo").build();
+    assertEquals("foo", entity.getString("string"));
+  }
+
+  @Test
+  public void testGetLong() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals(125, entity.getLong("long"));
+    entity = builder.set("long", LongValue.of(10)).build();
+    assertEquals(10, entity.getLong("long"));
+  }
+
+  @Test
+  public void testGetDouble() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals(1.25, entity.getDouble("double"), 0);
+    entity = builder.set("double", DoubleValue.of(10)).build();
+    assertEquals(10, entity.getDouble("double"), 0);
+  }
+
+  @Test
+  public void testGetBoolean() throws Exception {
+    BaseEntity entity = builder.build();
+    assertTrue(entity.getBoolean("boolean"));
+    entity = builder.set("boolean", BooleanValue.of(false)).build();
+    assertFalse(entity.getBoolean("boolean"));
+  }
+
+  @Test
+  public void testGetDateTime() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
+    Calendar cal = Calendar.getInstance();
+    cal.add(Calendar.DATE, -1);
+    DateTime dateTime = DateTime.copyFrom(cal);
+    entity = builder.set("dateTime", DateTimeValue.of(dateTime)).build();
+    assertEquals(dateTime, entity.getDateTime("dateTime"));
+  }
+
+  @Test
+  public void testGetKey() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals(KEY, entity.getKey("key"));
+    Key key = Key.builder(KEY).name("BLA").build();
+    entity = builder.set("key", key).build();
+    assertEquals(key, entity.getKey("key"));
+  }
+
+  @Test
+  public void testGetEntity() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals(ENTITY, entity.getEntity("entity"));
+    assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
+    entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
+    assertEquals(PARTIAL_ENTITY, entity.getEntity("entity"));
+  }
+
+  @Test
+  public void testGetList() throws Exception {
+    BaseEntity entity = builder.build();
+    List> list = entity.getList("list1");
+    assertEquals(2, list.size());
+    assertEquals(NullValue.of(), list.get(0));
+    assertEquals("foo", list.get(1).get());
+    list = entity.getList("list2");
+    assertEquals(2, list.size());
+    assertEquals(Long.valueOf(10), list.get(0).get());
+    assertEquals(Double.valueOf(2), list.get(1).get());
+    list = entity.getList("list3");
+    assertEquals(1, list.size());
+    assertEquals(Boolean.TRUE, list.get(0).get());
+    entity = builder.set("list1", ListValue.of(list)).build();
+    assertEquals(list, entity.getList("list1"));
+  }
+
+  @Test
+  public void testGetBlob() throws Exception {
+    BaseEntity entity = builder.build();
+    assertEquals(BLOB, entity.getBlob("blob"));
+    Blob blob = Blob.copyFrom(new byte[] {});
+    entity = builder.set("blob", BlobValue.of(blob)).build();
+    assertEquals(blob, entity.getBlob("blob"));
+  }
+
+  @Test
+  public void testNames() throws Exception {
+    Set names = ImmutableSet.builder()
+        .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
+        .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
+        .build();
+    BaseEntity entity = builder.build();
+    assertEquals(names, entity.names());
+  }
+}
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
index e78023101e48..9cd0b4ce332a 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
@@ -1 +1,111 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;

import org.junit.Before;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Random;

public class BlobTest {

  private byte[] bytes1 = new byte[10];
  private byte[] bytes2 = new byte[11];
  private Blob blob1;
  private Blob blob2;


  @Before
  public void setUp() {
    Random rnd = new Random();
    rnd.nextBytes(bytes1);
    rnd.nextBytes(bytes2);
    blob1 = Blob.copyFrom(bytes1);
    blob2 = Blob.copyFrom(bytes2);
  }

  @Test
  public void testEquals() throws Exception {
    assertEquals(blob1, blob1);
    assertEquals(blob1, Blob.copyFrom(bytes1));
    assertNotEquals(blob1, blob2);
  }

  @Test
  public void testLength() throws Exception {
    assertEquals(bytes1.length, blob1.length());
    assertEquals(bytes2.length, blob2.length());
  }

  @Test
  public void testToByteArray() throws Exception {
    assertArrayEquals(bytes1, blob1.toByteArray());
    assertArrayEquals(bytes2, blob2.toByteArray());
  }

  @Test
  public void testAsReadOnlyByteBuffer() throws Exception {
    ByteBuffer buffer = blob1.asReadOnlyByteBuffer();
    byte[] bytes = new byte[bytes1.length];
    buffer.get(bytes);
    assertFalse(buffer.hasRemaining());
    assertArrayEquals(bytes1, bytes);
  }

  @Test
  public void testAsInputStream() throws Exception {
    byte[] bytes = new byte[bytes1.length];
    InputStream in = blob1.asInputStream();
    assertEquals(bytes1.length, in.read(bytes));
    assertEquals(-1, in.read());
    assertArrayEquals(bytes1, bytes);
  }

  @Test
  public void testCopyTo() throws Exception {
    byte[] bytes = new byte[bytes1.length];
    blob1.copyTo(bytes);
    assertArrayEquals(bytes1, bytes);

    ByteBuffer buffer = ByteBuffer.allocate(bytes1.length);
    blob1.copyTo(buffer);
    buffer.flip();
    bytes = new byte[bytes1.length];
    buffer.get(bytes);
    assertFalse(buffer.hasRemaining());
    assertArrayEquals(bytes1, bytes);
  }

  @Test
  public void testCopyFrom() throws Exception {
    Blob blob = Blob.copyFrom(ByteBuffer.wrap(bytes1));
    assertEquals(blob1, blob);
    assertArrayEquals(bytes1, blob.toByteArray());

    blob = Blob.copyFrom(new ByteArrayInputStream(bytes2));
    assertEquals(blob2, blob);
    assertArrayEquals(bytes2, blob.toByteArray());
  }
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+public class BlobTest {
+
+  private byte[] bytes1 = new byte[10];
+  private byte[] bytes2 = new byte[11];
+  private Blob blob1;
+  private Blob blob2;
+
+
+  @Before
+  public void setUp() {
+    Random rnd = new Random();
+    rnd.nextBytes(bytes1);
+    rnd.nextBytes(bytes2);
+    blob1 = Blob.copyFrom(bytes1);
+    blob2 = Blob.copyFrom(bytes2);
+  }
+
+  @Test
+  public void testEquals() throws Exception {
+    assertEquals(blob1, blob1);
+    assertEquals(blob1, Blob.copyFrom(bytes1));
+    assertNotEquals(blob1, blob2);
+  }
+
+  @Test
+  public void testLength() throws Exception {
+    assertEquals(bytes1.length, blob1.length());
+    assertEquals(bytes2.length, blob2.length());
+  }
+
+  @Test
+  public void testToByteArray() throws Exception {
+    assertArrayEquals(bytes1, blob1.toByteArray());
+    assertArrayEquals(bytes2, blob2.toByteArray());
+  }
+
+  @Test
+  public void testAsReadOnlyByteBuffer() throws Exception {
+    ByteBuffer buffer = blob1.asReadOnlyByteBuffer();
+    byte[] bytes = new byte[bytes1.length];
+    buffer.get(bytes);
+    assertFalse(buffer.hasRemaining());
+    assertArrayEquals(bytes1, bytes);
+  }
+
+  @Test
+  public void testAsInputStream() throws Exception {
+    byte[] bytes = new byte[bytes1.length];
+    InputStream in = blob1.asInputStream();
+    assertEquals(bytes1.length, in.read(bytes));
+    assertEquals(-1, in.read());
+    assertArrayEquals(bytes1, bytes);
+  }
+
+  @Test
+  public void testCopyTo() throws Exception {
+    byte[] bytes = new byte[bytes1.length];
+    blob1.copyTo(bytes);
+    assertArrayEquals(bytes1, bytes);
+
+    ByteBuffer buffer = ByteBuffer.allocate(bytes1.length);
+    blob1.copyTo(buffer);
+    buffer.flip();
+    bytes = new byte[bytes1.length];
+    buffer.get(bytes);
+    assertFalse(buffer.hasRemaining());
+    assertArrayEquals(bytes1, bytes);
+  }
+
+  @Test
+  public void testCopyFrom() throws Exception {
+    Blob blob = Blob.copyFrom(ByteBuffer.wrap(bytes1));
+    assertEquals(blob1, blob);
+    assertArrayEquals(bytes1, blob.toByteArray());
+
+    blob = Blob.copyFrom(new ByteArrayInputStream(bytes2));
+    assertEquals(blob2, blob);
+    assertArrayEquals(bytes2, blob.toByteArray());
+  }
+}
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
index 6806fd698331..e4a9b6cbc742 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
@@ -1 +1,52 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;

import com.google.protobuf.ByteString;

import org.junit.Before;
import org.junit.Test;

public class CursorTest {

  private byte[] bytes1 = {1, 2, 3, '%', '<', '+'};
  private byte[] bytes2 = {10, 20, 30};
  private Cursor cursor1;
  private Cursor cursor2;

  @Before
  public void setUp() throws Exception {
    cursor1 = new Cursor(ByteString.copyFrom(bytes1));
    cursor2 = new Cursor(ByteString.copyFrom(bytes2));
  }

  @Test
  public void testToFromUrlSafe() throws Exception {
    String urlSafe = cursor1.toUrlSafe();
    assertEquals(cursor1, Cursor.fromUrlSafe(urlSafe));
  }

  @Test
  public void testCopyFrom() throws Exception {
    Cursor cursor = Cursor.copyFrom(bytes2);
    assertEquals(cursor2, cursor);
    assertNotEquals(cursor1, cursor);
  }
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import com.google.protobuf.ByteString;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class CursorTest {
+
+  private byte[] bytes1 = {1, 2, 3, '%', '<', '+'};
+  private byte[] bytes2 = {10, 20, 30};
+  private Cursor cursor1;
+  private Cursor cursor2;
+
+  @Before
+  public void setUp() throws Exception {
+    cursor1 = new Cursor(ByteString.copyFrom(bytes1));
+    cursor2 = new Cursor(ByteString.copyFrom(bytes2));
+  }
+
+  @Test
+  public void testToFromUrlSafe() throws Exception {
+    String urlSafe = cursor1.toUrlSafe();
+    assertEquals(cursor1, Cursor.fromUrlSafe(urlSafe));
+  }
+
+  @Test
+  public void testCopyFrom() throws Exception {
+    Cursor cursor = Cursor.copyFrom(bytes2);
+    assertEquals(cursor2, cursor);
+    assertNotEquals(cursor1, cursor);
+  }
+}
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java
index 1dd0f255ceca..a64a3531c19d 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DatastoreExceptionTest.java
@@ -1 +1,65 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;

import com.google.gcloud.datastore.DatastoreException.Code;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;

import org.junit.Test;

public class DatastoreExceptionTest {

  @Test
  public void testCode() throws Exception {
    for (Reason reason : Reason.values()) {
      Code code = Code.valueOf(reason.name());
      assertEquals(reason.retryable(), code.retryable());
      assertEquals(reason.description(), code.description());
      assertEquals(reason.httpStatus(), code.httpStatus());
    }

    DatastoreException exception = new DatastoreException(Code.ABORTED, "bla");
    assertEquals(Code.ABORTED, exception.code());
  }

  @Test
  public void testTranslateAndThrow() throws Exception {
    for (Reason reason : Reason.values()) {
      try {
        DatastoreException.translateAndThrow(new DatastoreRpcException(reason));
        fail("Exception expected");
      } catch (DatastoreException ex) {
        assertEquals(reason.name(), ex.code().name());
      }
    }
  }

  @Test
  public void testThrowInvalidRequest() throws Exception {
    try {
      DatastoreException.throwInvalidRequest("message %s %d", "a", 1);
      fail("Exception expected");
    } catch (DatastoreException ex) {
      assertEquals(Code.FAILED_PRECONDITION, ex.code());
      assertEquals("message a 1", ex.getMessage());
    }
  }
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.gcloud.datastore.DatastoreException.Code;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException;
+import com.google.gcloud.spi.DatastoreRpc.DatastoreRpcException.Reason;
+
+import org.junit.Test;
+
+public class DatastoreExceptionTest {
+
+  @Test
+  public void testCode() throws Exception {
+    for (Reason reason : Reason.values()) {
+      Code code = Code.valueOf(reason.name());
+      assertEquals(reason.retryable(), code.retryable());
+      assertEquals(reason.description(), code.description());
+      assertEquals(reason.httpStatus(), code.httpStatus());
+    }
+
+    DatastoreException exception = new DatastoreException(Code.ABORTED, "bla");
+    assertEquals(Code.ABORTED, exception.code());
+  }
+
+  @Test
+  public void testTranslateAndThrow() throws Exception {
+    for (Reason reason : Reason.values()) {
+      try {
+        DatastoreException.translateAndThrow(new DatastoreRpcException(reason));
+        fail("Exception expected");
+      } catch (DatastoreException ex) {
+        assertEquals(reason.name(), ex.code().name());
+      }
+    }
+  }
+
+  @Test
+  public void testThrowInvalidRequest() throws Exception {
+    try {
+      DatastoreException.throwInvalidRequest("message %s %d", "a", 1);
+      fail("Exception expected");
+    } catch (DatastoreException ex) {
+      assertEquals(Code.FAILED_PRECONDITION, ex.code());
+      assertEquals("message a 1", ex.getMessage());
+    }
+  }
+}
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
index a7131e04e89c..23f3951a5dc3 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/DateTimeTest.java
@@ -1 +1,77 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.datastore;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;

import org.junit.Test;

import java.util.Calendar;

public class DateTimeTest {

  @Test
  public void testTimestampMicroseconds() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis() * 1000, date.timestampMicroseconds());
  }

  @Test
  public void testTimestampMillis() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTimeInMillis(), date.timestampMillis());
  }

  @Test
  public void testToDate() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal.getTime(), date.toDate());
  }

  @Test
  public void testToCalendar() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date = DateTime.copyFrom(cal);
    assertEquals(cal, date.toCalendar());
  }

  @Test
  public void testNow() throws Exception {
    Calendar cal1 = Calendar.getInstance();
    DateTime now = DateTime.now();
    Calendar cal2 = Calendar.getInstance();
    assertTrue(now.timestampMillis() >= cal1.getTimeInMillis());
    assertTrue(now.timestampMillis() <= cal2.getTimeInMillis());
  }

  @Test
  public void testCopyFrom() throws Exception {
    Calendar cal = Calendar.getInstance();
    DateTime date1 = DateTime.copyFrom(cal);
    DateTime date2 = DateTime.copyFrom(cal.getTime());
    cal.add(Calendar.DATE, 1);
    DateTime date3 = DateTime.copyFrom(cal.getTime());
    assertEquals(date1, date2);
    assertNotEquals(date1, date3);
  }
}

\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.datastore;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+import java.util.Calendar;
+
+public class DateTimeTest {
+
+  @Test
+  public void testTimestampMicroseconds() throws Exception {
+    Calendar cal = Calendar.getInstance();
+    DateTime date = DateTime.copyFrom(cal);
+    assertEquals(cal.getTimeInMillis() * 1000, date.timestampMicroseconds());
+  }
+
+  @Test
+  public void testTimestampMillis() throws Exception {
+    Calendar cal = Calendar.getInstance();
+    DateTime date = DateTime.copyFrom(cal);
+    assertEquals(cal.getTimeInMillis(), date.timestampMillis());
+  }
+
+  @Test
+  public void testToDate() throws Exception {
+    Calendar cal = Calendar.getInstance();
+    DateTime date = DateTime.copyFrom(cal);
+    assertEquals(cal.getTime(), date.toDate());
+  }
+
+  @Test
+  public void testToCalendar() throws Exception {
+    Calendar cal = Calendar.getInstance();
+    DateTime date = DateTime.copyFrom(cal);
+    assertEquals(cal, date.toCalendar());
+  }
+
+  @Test
+  public void testNow() throws Exception {
+    Calendar cal1 = Calendar.getInstance();
+    DateTime now = DateTime.now();
+    Calendar cal2 = Calendar.getInstance();
+    assertTrue(now.timestampMillis() >= cal1.getTimeInMillis());
+    assertTrue(now.timestampMillis() <= cal2.getTimeInMillis());
+  }
+
+  @Test
+  public void testCopyFrom() throws Exception {
+    Calendar cal = Calendar.getInstance();
+    DateTime date1 = DateTime.copyFrom(cal);
+    DateTime date2 = DateTime.copyFrom(cal.getTime());
+    cal.add(Calendar.DATE, 1);
+    DateTime date3 = DateTime.copyFrom(cal.getTime());
+    assertEquals(date1, date2);
+    assertNotEquals(date1, date3);
+  }
+}
+
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
index bd5d9b782975..7448fec9d32f 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java
@@ -1 +1,171 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.spi;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gcloud.storage.StorageException;

import java.util.List;
import java.util.Map;

public interface StorageRpc {

  // These options are part of the Google Cloud storage header options
  enum Option {
    PREDEFINED_ACL("predefinedAcl"),
    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
    IF_GENERATION_MATCH("ifGenerationNotMatch"),
    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
    PREFIX("prefix"),
    MAX_RESULTS("maxResults"),
    PAGE_TOKEN("pageToken"),
    DELIMITER("delimiter"),
    VERSIONS("versions");

    private final String value;

    Option(String value) {
      this.value = value;
    }

    public String value() {
      return value;
    }

    @SuppressWarnings("unchecked")
     T get(Map options) {
      return (T) options.get(this);
    }

    String getString(Map options) {
      return get(options);
    }

    Long getLong(Map options) {
      return get(options);
    }

    Boolean getBoolean(Map options) {
      return get(options);
    }
  }

  class Tuple {

    private final X x;
    private final Y y;

    private Tuple(X x, Y y) {
      this.x = x;
      this.y = y;
    }

    public static  Tuple of(X x, Y y) {
      return new Tuple<>(x, y);
    }

    public X x() {
      return x;
    }

    public Y y() {
      return y;
    }
  }

  class BatchRequest {

    public final List>> toDelete;
    public final List>> toUpdate;
    public final List>> toGet;

    public BatchRequest(Iterable>> toDelete,
        Iterable>> toUpdate,
        Iterable>> toGet) {
      this.toDelete = ImmutableList.copyOf(toDelete);
      this.toUpdate = ImmutableList.copyOf(toUpdate);
      this.toGet = ImmutableList.copyOf(toGet);
    }
  }

  class BatchResponse {

    public final Map> deletes;
    public final Map> updates;
    public final Map> gets;

    public BatchResponse(Map> deletes,
        Map> updates,
        Map> gets) {
      this.deletes = ImmutableMap.copyOf(deletes);
      this.updates = ImmutableMap.copyOf(updates);
      this.gets = ImmutableMap.copyOf(gets);
    }
  }

  Bucket create(Bucket bucket, Map options) throws StorageException;

  StorageObject create(StorageObject object, byte[] content, Map options)
      throws StorageException;

  Tuple> list(Map options) throws StorageException;

  Tuple> list(String bucket, Map options)
      throws StorageException;

  Bucket get(Bucket bucket, Map options) throws StorageException;

  StorageObject get(StorageObject object, Map options)
      throws StorageException;

  Bucket patch(Bucket bucket, Map options) throws StorageException;

  StorageObject patch(StorageObject storageObject, Map options)
      throws StorageException;

  boolean delete(Bucket bucket, Map options) throws StorageException;

  boolean delete(StorageObject object, Map options) throws StorageException;

  BatchResponse batch(BatchRequest request) throws StorageException;

  StorageObject compose(Iterable sources, StorageObject target,
      Map targetOptions) throws StorageException;

  StorageObject copy(StorageObject source, Map sourceOptions,
      StorageObject target, Map targetOptions) throws StorageException;

  byte[] load(StorageObject storageObject, Map options)
      throws StorageException;

  byte[] read(StorageObject from, Map options, long position, int bytes)
      throws StorageException;

  String open(StorageObject object, Map options) throws StorageException;

  void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
      long destOffset, int length, boolean last) throws StorageException;
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.spi;
+
+import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.gcloud.storage.StorageException;
+
+import java.util.List;
+import java.util.Map;
+
+public interface StorageRpc {
+
+  // These options are part of the Google Cloud storage header options
+  enum Option {
+    PREDEFINED_ACL("predefinedAcl"),
+    PREDEFINED_DEFAULT_OBJECT_ACL("predefinedDefaultObjectAcl"),
+    IF_METAGENERATION_MATCH("ifMetagenerationMatch"),
+    IF_METAGENERATION_NOT_MATCH("ifMetagenerationNotMatch"),
+    IF_GENERATION_NOT_MATCH("ifGenerationMatch"),
+    IF_GENERATION_MATCH("ifGenerationNotMatch"),
+    IF_SOURCE_METAGENERATION_MATCH("ifSourceMetagenerationMatch"),
+    IF_SOURCE_METAGENERATION_NOT_MATCH("ifSourceMetagenerationNotMatch"),
+    IF_SOURCE_GENERATION_MATCH("ifSourceGenerationMatch"),
+    IF_SOURCE_GENERATION_NOT_MATCH("ifSourceGenerationNotMatch"),
+    PREFIX("prefix"),
+    MAX_RESULTS("maxResults"),
+    PAGE_TOKEN("pageToken"),
+    DELIMITER("delimiter"),
+    VERSIONS("versions");
+
+    private final String value;
+
+    Option(String value) {
+      this.value = value;
+    }
+
+    public String value() {
+      return value;
+    }
+
+    @SuppressWarnings("unchecked")
+     T get(Map options) {
+      return (T) options.get(this);
+    }
+
+    String getString(Map options) {
+      return get(options);
+    }
+
+    Long getLong(Map options) {
+      return get(options);
+    }
+
+    Boolean getBoolean(Map options) {
+      return get(options);
+    }
+  }
+
+  class Tuple {
+
+    private final X x;
+    private final Y y;
+
+    private Tuple(X x, Y y) {
+      this.x = x;
+      this.y = y;
+    }
+
+    public static  Tuple of(X x, Y y) {
+      return new Tuple<>(x, y);
+    }
+
+    public X x() {
+      return x;
+    }
+
+    public Y y() {
+      return y;
+    }
+  }
+
+  class BatchRequest {
+
+    public final List>> toDelete;
+    public final List>> toUpdate;
+    public final List>> toGet;
+
+    public BatchRequest(Iterable>> toDelete,
+        Iterable>> toUpdate,
+        Iterable>> toGet) {
+      this.toDelete = ImmutableList.copyOf(toDelete);
+      this.toUpdate = ImmutableList.copyOf(toUpdate);
+      this.toGet = ImmutableList.copyOf(toGet);
+    }
+  }
+
+  class BatchResponse {
+
+    public final Map> deletes;
+    public final Map> updates;
+    public final Map> gets;
+
+    public BatchResponse(Map> deletes,
+        Map> updates,
+        Map> gets) {
+      this.deletes = ImmutableMap.copyOf(deletes);
+      this.updates = ImmutableMap.copyOf(updates);
+      this.gets = ImmutableMap.copyOf(gets);
+    }
+  }
+
+  Bucket create(Bucket bucket, Map options) throws StorageException;
+
+  StorageObject create(StorageObject object, byte[] content, Map options)
+      throws StorageException;
+
+  Tuple> list(Map options) throws StorageException;
+
+  Tuple> list(String bucket, Map options)
+      throws StorageException;
+
+  Bucket get(Bucket bucket, Map options) throws StorageException;
+
+  StorageObject get(StorageObject object, Map options)
+      throws StorageException;
+
+  Bucket patch(Bucket bucket, Map options) throws StorageException;
+
+  StorageObject patch(StorageObject storageObject, Map options)
+      throws StorageException;
+
+  boolean delete(Bucket bucket, Map options) throws StorageException;
+
+  boolean delete(StorageObject object, Map options) throws StorageException;
+
+  BatchResponse batch(BatchRequest request) throws StorageException;
+
+  StorageObject compose(Iterable sources, StorageObject target,
+      Map targetOptions) throws StorageException;
+
+  StorageObject copy(StorageObject source, Map sourceOptions,
+      StorageObject target, Map targetOptions) throws StorageException;
+
+  byte[] load(StorageObject storageObject, Map options)
+      throws StorageException;
+
+  byte[] read(StorageObject from, Map options, long position, int bytes)
+      throws StorageException;
+
+  String open(StorageObject object, Map options) throws StorageException;
+
+  void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
+      long destOffset, int length, boolean last) throws StorageException;
+}
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageException.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageException.java
index 9bf073bef6b4..45734d631aec 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageException.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageException.java
@@ -1 +1,48 @@
-/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.gcloud.storage;

/**
 * Storage service exception.
 *
 * @see Google Cloud
 *      Storage error codes
 */
public class StorageException extends RuntimeException {

  private static final long serialVersionUID = -3748432005065428084L;

  private final int code;
  private final boolean retryable;

  public StorageException(int code, String message, boolean retryable) {
    super(message);
    this.code = code;
    this.retryable = retryable;
  }

  /**
   * Returns the code associated with this exception.
   */
  public int code() {
    return code;
  }

  public boolean retryable() {
    return retryable;
  }
}
\ No newline at end of file
+/*
+ * Copyright 2015 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.gcloud.storage;
+
+/**
+ * Storage service exception.
+ *
+ * @see Google Cloud
+ *      Storage error codes
+ */
+public class StorageException extends RuntimeException {
+
+  private static final long serialVersionUID = -3748432005065428084L;
+
+  private final int code;
+  private final boolean retryable;
+
+  public StorageException(int code, String message, boolean retryable) {
+    super(message);
+    this.code = code;
+    this.retryable = retryable;
+  }
+
+  /**
+   * Returns the code associated with this exception.
+   */
+  public int code() {
+    return code;
+  }
+
+  public boolean retryable() {
+    return retryable;
+  }
+}

From d15f2ca71c9723a1323df40d63da708996e3176b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89amonn=20McManus?= 
Date: Fri, 5 Jun 2015 17:15:11 -0700
Subject: [PATCH 258/732] Minor cleanups. No behaviour or API changes are
 intended, except for a few types which were made public because they are
 referenced in the signatures of public methods.

* Use the NameT convention for some type variables for improved readability.

* Avoid using raw types where possible.

* Limit the scope of @SuppressWarnings where possible.

* Split some lines that were over the 100-character limit.
---
 .../com/google/gcloud/AuthCredentials.java    |  7 ++-
 .../main/java/com/google/gcloud/Service.java  |  4 +-
 .../com/google/gcloud/ServiceOptions.java     | 45 +++++++++++--------
 .../google/gcloud/spi/ServiceRpcFactory.java  |  7 ++-
 .../google/gcloud/datastore/BaseEntity.java   | 19 ++++----
 .../google/gcloud/datastore/BatchImpl.java    |  2 +-
 .../google/gcloud/datastore/Datastore.java    |  2 +-
 .../gcloud/datastore/DatastoreException.java  |  3 +-
 .../gcloud/datastore/DatastoreImpl.java       | 19 +++-----
 .../gcloud/datastore/DatastoreOptions.java    |  4 +-
 .../com/google/gcloud/datastore/DateTime.java |  8 ++--
 .../com/google/gcloud/datastore/Entity.java   |  2 +-
 .../google/gcloud/datastore/EntityValue.java  | 18 ++++----
 .../google/gcloud/datastore/FullEntity.java   |  6 +--
 .../com/google/gcloud/datastore/GqlQuery.java | 21 ++++-----
 .../java/com/google/gcloud/datastore/Key.java |  3 +-
 .../google/gcloud/datastore/KeyFactory.java   |  2 +-
 .../gcloud/datastore/ProjectionEntity.java    |  8 ++--
 .../com/google/gcloud/datastore/Query.java    |  5 +--
 .../google/gcloud/datastore/QueryResults.java |  4 +-
 .../gcloud/datastore/QueryResultsImpl.java    |  5 ++-
 .../com/google/gcloud/datastore/RawValue.java |  3 +-
 .../gcloud/datastore/StructuredQuery.java     |  3 +-
 .../gcloud/datastore/TransactionImpl.java     |  2 +-
 .../google/gcloud/datastore/ValueBuilder.java |  2 +-
 .../google/gcloud/datastore/ValueType.java    |  3 +-
 .../com/google/gcloud/spi/DatastoreRpc.java   |  3 +-
 .../BaseDatastoreBatchWriterTest.java         |  5 ++-
 .../gcloud/datastore/BaseEntityTest.java      | 38 ++++++++--------
 .../com/google/gcloud/datastore/BlobTest.java |  4 +-
 .../google/gcloud/datastore/CursorTest.java   |  4 +-
 .../google/gcloud/datastore/EntityTest.java   |  4 +-
 .../gcloud/datastore/LocalGcdHelper.java      |  8 ++--
 .../datastore/ProjectionEntityTest.java       |  3 +-
 .../gcloud/datastore/SerializationTest.java   | 12 ++---
 .../google/gcloud/datastore/ValueTest.java    |  1 +
 .../gcloud/examples/DatastoreExample.java     |  9 ++--
 .../java/com/google/gcloud/storage/Acl.java   |  2 +-
 .../google/gcloud/storage/BatchResponse.java  |  7 +--
 .../gcloud/storage/BlobWriterChannelImpl.java |  2 +-
 .../com/google/gcloud/storage/BucketInfo.java |  8 +++-
 .../com/google/gcloud/storage/ListResult.java |  2 +-
 .../com/google/gcloud/storage/Storage.java    |  3 +-
 .../google/gcloud/storage/StorageOptions.java |  2 +-
 .../google/gcloud/storage/ListResultTest.java |  6 +--
 .../gcloud/storage/SerializationTest.java     | 12 ++---
 .../com/google/gcloud/storage/HttpMethod.java | 24 ----------
 47 files changed, 184 insertions(+), 182 deletions(-)
 delete mode 100644 src/main/java/com/google/gcloud/storage/HttpMethod.java

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
index 6cdb737ddd91..13d91836ae4c 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/AuthCredentials.java
@@ -181,10 +181,9 @@ public static AuthCredentials createForComputeEngine()
   /**
    * Returns the Application Default Credentials.
    *
-   * 
    
-   * Returns the Application Default Credentials which are credentials that identify and authorize
-   * the whole application. This is the built-in service account if running on Google Compute Engine
-   * or the credentials file from the path in the environment variable
+   * 
    Returns the Application Default Credentials which are credentials that identify and
+   * authorize the whole application. This is the built-in service account if running on
+   * Google Compute Engine or the credentials file from the path in the environment variable
    * GOOGLE_APPLICATION_CREDENTIALS.
    * 
    
    *
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/Service.java b/gcloud-java-core/src/main/java/com/google/gcloud/Service.java
index 8a43b9cf2244..19759fb20e21 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/Service.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/Service.java
@@ -16,6 +16,6 @@
 
 package com.google.gcloud;
 
-public interface Service {
-  O options();
+public interface Service> {
+  OptionsT options();
 }
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
index a974a1f1912a..cd529ba14187 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/ServiceOptions.java
@@ -45,7 +45,10 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-public abstract class ServiceOptions> implements Serializable {
+public abstract class ServiceOptions<
+        ServiceRpcT,
+        OptionsT extends ServiceOptions>
+    implements Serializable {
 
   private static final String DEFAULT_HOST = "https://www.googleapis.com";
   private static final long serialVersionUID = 1203687993961393350L;
@@ -56,7 +59,7 @@ public abstract class ServiceOptions> implemen
   private final HttpTransportFactory httpTransportFactory;
   private final AuthCredentials authCredentials;
   private final RetryParams retryParams;
-  private final ServiceRpcFactory serviceRpcFactory;
+  private final ServiceRpcFactory serviceRpcFactory;
 
   public interface HttpTransportFactory extends Serializable {
     HttpTransport create();
@@ -88,19 +91,21 @@ public HttpTransport create() {
 
 
 
-  protected abstract static class Builder,
-      B extends Builder> {
+  protected abstract static class Builder<
+      ServiceRpcT,
+      OptionsT extends ServiceOptions,
+      B extends Builder> {
 
     private String projectId;
     private String host;
     private HttpTransportFactory httpTransportFactory;
     private AuthCredentials authCredentials;
     private RetryParams retryParams;
-    private ServiceRpcFactory serviceRpcFactory;
+    private ServiceRpcFactory serviceRpcFactory;
 
     protected Builder() {}
 
-    protected Builder(ServiceOptions options) {
+    protected Builder(ServiceOptions options) {
       projectId = options.projectId;
       host = options.host;
       httpTransportFactory = options.httpTransportFactory;
@@ -109,7 +114,7 @@ protected Builder(ServiceOptions options) {
       serviceRpcFactory = options.serviceRpcFactory;
     }
 
-    protected abstract ServiceOptions build();
+    protected abstract ServiceOptions build();
 
     @SuppressWarnings("unchecked")
     protected B self() {
@@ -141,13 +146,13 @@ public B retryParams(RetryParams retryParams) {
       return self();
     }
 
-    public B serviceRpcFactory(ServiceRpcFactory serviceRpcFactory) {
+    public B serviceRpcFactory(ServiceRpcFactory serviceRpcFactory) {
       this.serviceRpcFactory = serviceRpcFactory;
       return self();
     }
   }
 
-  protected ServiceOptions(Builder builder) {
+  protected ServiceOptions(Builder builder) {
     projectId = checkNotNull(builder.projectId != null ? builder.projectId : defaultProject());
     host = firstNonNull(builder.host, DEFAULT_HOST);
     httpTransportFactory =
@@ -222,7 +227,7 @@ protected static String googleCloudProjectId() {
       String section = null;
       Pattern projectPattern = Pattern.compile("^project\\s*=\\s*(.*)$");
       Pattern sectionPattern = Pattern.compile("^\\[(.*)\\]$");
-      while((line = reader.readLine()) != null) {
+      while ((line = reader.readLine()) != null) {
         if (line.isEmpty() || line.startsWith(";")) {
           continue;
         }
@@ -245,7 +250,7 @@ protected static String googleCloudProjectId() {
   }
 
   private static boolean isWindows() {
-    return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).indexOf("windows") > -1;
+    return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("windows");
   }
 
   protected static String getAppEngineProjectId() {
@@ -288,7 +293,7 @@ public RetryParams retryParams() {
     return retryParams != null ? retryParams : RetryParams.noRetries();
   }
 
-  public ServiceRpcFactory serviceRpcFactory() {
+  public ServiceRpcFactory serviceRpcFactory() {
     return serviceRpcFactory;
   }
 
@@ -297,13 +302,12 @@ public HttpRequestInitializer httpRequestInitializer() {
     return authCredentials().httpRequestInitializer(httpTransport, scopes());
   }
 
-  @Override
-  public int hashCode() {
+  protected int baseHashCode() {
     return Objects.hash(projectId, host, httpTransportFactory, authCredentials, retryParams,
         serviceRpcFactory);
   }
 
-  protected boolean isEquals(ServiceOptions other) {
+  protected boolean baseEquals(ServiceOptions other) {
     return Objects.equals(projectId, other.projectId)
         && Objects.equals(host, other.host)
         && Objects.equals(httpTransportFactory, other.httpTransportFactory)
@@ -312,14 +316,17 @@ protected boolean isEquals(ServiceOptions other) {
         && Objects.equals(serviceRpcFactory, other.serviceRpcFactory);
   }
 
-  public abstract Builder toBuilder();
+  public abstract Builder toBuilder();
 
   /**
    * Creates a service RPC using a factory loaded by {@link ServiceLoader}.
    */
-  protected static > R createRpc(O options,
-      Class> factoryClass) {
-    ServiceRpcFactory factory = Iterables.getFirst(ServiceLoader.load(factoryClass), null);
+  protected static
+      >
+      ServiceRpcT createRpc(OptionsT options,
+          Class> factoryClass) {
+    ServiceRpcFactory factory =
+        Iterables.getFirst(ServiceLoader.load(factoryClass), null);
     return factory == null ? null : factory.create(options);
   }
 }
diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
index 9bab3f81ce9c..89e08cda9eda 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/spi/ServiceRpcFactory.java
@@ -24,7 +24,10 @@
  * A base interface for all service RPC factories.
  * Loading of a factory implementation is done via {@link java.util.ServiceLoader}.
  */
-public interface ServiceRpcFactory extends Serializable {
+public interface ServiceRpcFactory<
+        ServiceRpcT,
+        OptionsT extends ServiceOptions>
+    extends Serializable {
 
-  S create(O options);
+  ServiceRpcT create(OptionsT options);
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
index 21e20b33a8ed..3a79f3053a1e 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BaseEntity.java
@@ -40,12 +40,13 @@
 
 /**
  * A base class for entities (key and properties).
- * An entity is Google Cloud Datastore persistent data object.
+ * An entity is a Google Cloud Datastore persistent data object.
  * An entity holds one or more properties, represented by a name (as {@link String})
  * and a value (as {@link com.google.gcloud.datastore.Value}), and may be associated with a
  * key. For a list of possible values see {@link ValueType}.
  *
- * @see Google Cloud Datastore Entities, Properties, and Keys
+ * @see Google Cloud Datastore
+ *     Entities, Properties, and Keys
  */
 public abstract class BaseEntity extends Serializable {
 
@@ -54,7 +55,7 @@ public abstract class BaseEntity extends Serializable> properties;
   private final K key;
 
-  abstract static class Builder> {
+  public abstract static class Builder> {
 
     private K key;
     private final Map> properties = new HashMap<>();
@@ -127,7 +128,7 @@ public B remove(String name) {
       return self();
     }
 
-    public B set(String name, Value value) {
+    public B set(String name, Value value) {
       properties.put(name, value);
       return self();
     }
@@ -187,7 +188,7 @@ public B setNull(String name) {
       return self();
     }
 
-    public abstract BaseEntity build();
+    public abstract BaseEntity build();
   }
 
   BaseEntity(Builder builder) {
@@ -213,7 +214,7 @@ public boolean equals(Object obj) {
     if (!(obj instanceof BaseEntity)) {
       return false;
     }
-    BaseEntity other = (BaseEntity) obj;
+    BaseEntity other = (BaseEntity) obj;
     return Objects.equals(key, other.key)
         && Objects.equals(properties, other.properties);
   }
@@ -254,7 +255,7 @@ public > V getValue(String name) {
   }
 
   /**
-   * Returns true if property is instanceof NullValue.
+   * Returns true if property is an instance of NullValue.
    *
    * @throws DatastoreException if not such property.
    */
@@ -375,12 +376,12 @@ ImmutableSortedMap> properties() {
 
   @Override
   protected Object fromPb(byte[] bytesPb) throws InvalidProtocolBufferException {
-    Builder builder = emptyBuilder();
+    Builder builder = emptyBuilder();
     builder.fill(DatastoreV1.Entity.parseFrom(bytesPb));
     return builder.build();
   }
 
-  protected abstract Builder emptyBuilder();
+  protected abstract Builder emptyBuilder();
 
   @Override
   protected final DatastoreV1.Entity toPb() {
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
index 9c95949e2c8a..8cb41304500b 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/BatchImpl.java
@@ -34,7 +34,7 @@ static class ResponseImpl implements Batch.Response {
 
     private final DatastoreV1.CommitResponse response;
 
-    public ResponseImpl(DatastoreV1.CommitResponse response) {
+    ResponseImpl(DatastoreV1.CommitResponse response) {
       this.response = response;
     }
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java
index fe79fdf45ff4..870ed8d9474f 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Datastore.java
@@ -34,7 +34,7 @@ public interface Datastore extends Service, DatastoreReaderWri
 
 
   /**
-   * An Callback for running with a Transactional
+   * A callback for running with a transactional
    * {@link com.google.gcloud.datastore.DatastoreReaderWriter}.
    * The associated transaction will be committed after a successful return from the {@code run}
    * method. Any propagated exception will cause the transaction to be rolled-back.
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
index d91cc2ccd98b..562578a26428 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreException.java
@@ -37,7 +37,8 @@ public class DatastoreException extends RuntimeException {
   /**
    * An error code to represent the failure.
    *
-   * @see Google Cloud Datastore error codes
+   * @see Google Cloud
+   *     Datastore error codes
    */
   public enum Code {
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java
index e848dd5e56c8..6f2454c62167 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreImpl.java
@@ -17,12 +17,10 @@
 package com.google.gcloud.datastore;
 
 import com.google.api.services.datastore.DatastoreV1;
-import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterators;
 import com.google.common.collect.Sets;
 import com.google.gcloud.BaseService;
 import com.google.gcloud.ExceptionHandler;
@@ -131,15 +129,11 @@ public List allocateId(IncompleteKey... keys) {
       requestPb.addKey(trimNameOrId(key).toPb());
     }
     DatastoreV1.AllocateIdsResponse responsePb = allocateIds(requestPb.build());
-    Iterator keyIterator = responsePb.getKeyList().iterator();
-    ImmutableList.Builder builder = ImmutableList.builder().addAll(
-        Iterators.transform(keyIterator, new Function() {
-          @Override
-          public Key apply(DatastoreV1.Key keyPb) {
-            return Key.fromPb(keyPb);
-          }
-        }));
-    return builder.build();
+    ImmutableList.Builder keyList = ImmutableList.builder();
+    for (DatastoreV1.Key keyPb : responsePb.getKeyList()) {
+      keyList.add(Key.fromPb(keyPb));
+    }
+    return keyList.build();
   }
 
   DatastoreV1.AllocateIdsResponse allocateIds(final DatastoreV1.AllocateIdsRequest requestPb) {
@@ -256,9 +250,6 @@ private void loadResults() {
     @SuppressWarnings("unchecked")
     @Override
     protected Entity computeNext() {
-      if (iter.hasNext()) {
-        return Entity.fromPb(iter.next().getEntity());
-      }
       while (!iter.hasNext()) {
         if (requestPb.getKeyCount() == 0) {
           return endOfData();
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
index ed6b51458938..c670bb5d9a2d 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DatastoreOptions.java
@@ -165,7 +165,7 @@ public Builder toBuilder() {
 
   @Override
   public int hashCode() {
-    return super.hashCode() ^ Objects.hash(namespace, force, normalizeDataset);
+    return baseHashCode() ^ Objects.hash(namespace, force, normalizeDataset);
   }
 
   @Override
@@ -174,7 +174,7 @@ public boolean equals(Object obj) {
       return false;
     }
     DatastoreOptions other = (DatastoreOptions) obj;
-    return isEquals(other) && Objects.equals(namespace, other.namespace)
+    return baseEquals(other) && Objects.equals(namespace, other.namespace)
         && Objects.equals(force, other.force)
         && Objects.equals(normalizeDataset, other.normalizeDataset);
   }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java
index 853856c1b696..af5a17ef7ef3 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/DateTime.java
@@ -31,7 +31,8 @@
  * A Google Cloud Datastore timestamp (represented in micro-seconds).
  * This class is immutable.
  *
- * @see Google Cloud Datastore Entities, Properties, and Keys
+ * @see Google Cloud Datastore
+ *     Entities, Properties, and Keys
  */
 public final class DateTime extends Serializable
     implements Comparable {
@@ -61,8 +62,9 @@ public int compareTo(DateTime other) {
 
   @Override
   public boolean equals(Object obj) {
-    return obj == this || obj instanceof DateTime
-        && timestampMicroseconds == ((DateTime) obj).timestampMicroseconds;
+    return obj == this
+        || (obj instanceof DateTime
+            && timestampMicroseconds == ((DateTime) obj).timestampMicroseconds);
   }
 
   public long timestampMicroseconds() {
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java
index 7842fba61f0c..dc1af5b8a2d9 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Entity.java
@@ -70,7 +70,7 @@ public Entity build() {
   }
 
   @Override
-  protected BaseEntity.Builder emptyBuilder() {
+  protected BaseEntity.Builder emptyBuilder() {
     return new Builder();
   }
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java
index add50e1747b3..da32c8eb2462 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/EntityValue.java
@@ -21,12 +21,12 @@
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.base.Preconditions;
 
-public class EntityValue extends Value {
+public class EntityValue extends Value> {
 
   private static final long serialVersionUID = -5461475706792576395L;
 
-  static final BaseMarshaller MARSHALLER =
-      new BaseMarshaller() {
+  static final BaseMarshaller, EntityValue, Builder> MARSHALLER =
+      new BaseMarshaller, EntityValue, Builder>() {
 
         private static final long serialVersionUID = 2355075086076070931L;
 
@@ -36,12 +36,12 @@ public int getProtoFieldId() {
         }
 
         @Override
-        public Builder newBuilder(FullEntity value) {
+        public Builder newBuilder(FullEntity value) {
           return builder(value);
         }
 
         @Override
-        protected FullEntity getValue(DatastoreV1.Value from) {
+        protected FullEntity getValue(DatastoreV1.Value from) {
           return FullEntity.fromPb(from.getEntityValue());
         }
 
@@ -51,7 +51,7 @@ protected void setValue(EntityValue from, DatastoreV1.Value.Builder to) {
         }
       };
 
-  public static final class Builder extends Value.BaseBuilder {
+  public static final class Builder extends Value.BaseBuilder, EntityValue, Builder> {
 
     private Builder() {
       super(ValueType.ENTITY);
@@ -70,7 +70,7 @@ public EntityValue build() {
     }
   }
 
-  public EntityValue(FullEntity entity) {
+  public EntityValue(FullEntity entity) {
     this(builder(entity));
   }
 
@@ -83,11 +83,11 @@ public Builder toBuilder() {
     return new Builder().mergeFrom(this);
   }
 
-  public static EntityValue of(FullEntity entity) {
+  public static EntityValue of(FullEntity entity) {
     return new EntityValue(entity);
   }
 
-  public static Builder builder(FullEntity entity) {
+  public static Builder builder(FullEntity entity) {
     return new Builder().set(entity).indexed(false);
   }
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
index d7084420665e..bb08fca12e3c 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/FullEntity.java
@@ -53,8 +53,8 @@ public FullEntity build() {
   }
 
   @Override
-  protected BaseEntity.Builder emptyBuilder() {
-    return new Builder();
+  protected BaseEntity.Builder emptyBuilder() {
+    return new Builder();
   }
 
   public static Builder builder() {
@@ -70,7 +70,7 @@ public static  Builder builder(FullEntity copyFro
   }
 
 
-  static FullEntity fromPb(DatastoreV1.Entity entityPb) {
+  static FullEntity fromPb(DatastoreV1.Entity entityPb) {
     return new Builder<>().fill(entityPb).build();
   }
 }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java
index 60c22637da56..e9bd8e12cfd8 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/GqlQuery.java
@@ -38,7 +38,7 @@
 import java.util.TreeMap;
 
 /**
- * A Google Cloud Datastore GQL.
+ * A Google Cloud Datastore GQL query.
  *
  * 
    A usage example:
    
  *
@@ -121,8 +121,8 @@ public boolean equals(Object obj) {
       }
       Binding other = (Binding) obj;
       return Objects.equals(name, other.name)
-        && Objects.equals(cursor, other.cursor)
-        && Objects.equals(value, other.value);
+          && Objects.equals(cursor, other.cursor)
+          && Objects.equals(value, other.value);
     }
 
     @Override
@@ -286,17 +286,17 @@ public GqlQuery build() {
       return new GqlQuery<>(this);
     }
 
-    @SuppressWarnings("rawtypes")
-    private static Binding toBinding(Value.BuilderFactory builderFactory, List values) {
+    private static Binding toBinding(Value.BuilderFactory builderFactory, List values) {
       return toBinding(null, builderFactory, values);
     }
 
-    @SuppressWarnings({"unchecked", "rawtypes"})
-    private static Binding toBinding(String name, Value.BuilderFactory builderFactory,
+    private static  Binding toBinding(String name, Value.BuilderFactory builderFactory,
         List values) {
-      List> list = new ArrayList<>(values.size());
+      List> list = new ArrayList<>(values.size());
       for (Object object : values) {
-        list.add(builderFactory.newBuilder(object).build());
+        @SuppressWarnings("unchecked")
+        V v = (V) object;
+        list.add(builderFactory.newBuilder(v).build());
       }
       Value value;
       if (list.isEmpty()) {
@@ -400,7 +400,8 @@ protected Object fromPb(ResultType resultType, String namespace, byte[] bytes
     return fromPb(resultType, namespace, DatastoreV1.GqlQuery.parseFrom(bytesPb));
   }
 
-  private static  GqlQuery fromPb(ResultType resultType, String ns, DatastoreV1.GqlQuery queryPb) {
+  private static  GqlQuery fromPb(
+      ResultType resultType, String ns, DatastoreV1.GqlQuery queryPb) {
     Builder builder = new Builder<>(resultType, queryPb.getQueryString());
     builder.namespace(ns);
     if (queryPb.hasAllowLiteral()) {
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Key.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Key.java
index 8b04898ffcd0..c625c067f6c2 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Key.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Key.java
@@ -33,7 +33,8 @@
  * Google Cloud Datastore {@link Entity}.
  * This class is immutable.
  *
- * @see Google Cloud Datastore Entities, Properties, and Keys
+ * @see Google Cloud Datastore
+ *    Entities, Properties, and Keys
  */
 public final class Key extends IncompleteKey {
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
index 8010468c5068..28f852ed5355 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/KeyFactory.java
@@ -19,7 +19,7 @@
 import com.google.common.collect.ImmutableList;
 
 /**
- * An helper for creating keys for a specific {@link Datastore},
+ * A helper for creating keys for a specific {@link Datastore},
  * using its associated projectId and namespace.
  */
 public final class KeyFactory extends BaseKey.Builder {
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
index acd9783fe836..1ba054b68161 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ProjectionEntity.java
@@ -24,14 +24,16 @@
  * A projection entity holds one or more properties, represented by a name (as {@link String})
  * and a value (as {@link Value}), and may have a {@link Key}.
  *
- * @see Google Cloud Datastore projection queries
- * @see Google Cloud Datastore Entities, Properties, and Keys
+ * @see Google Cloud
+ *     Datastore projection queries
+ * @see Google Cloud Datastore
+ *     Entities, Properties, and Keys
  */
 public final class ProjectionEntity extends BaseEntity {
 
   private static final long serialVersionUID = 432961565733066915L;
 
-  static final class Builder extends BaseEntity.Builder {
+  public static final class Builder extends BaseEntity.Builder {
 
     Builder() {
     }
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
index 093dc7283327..343535d94628 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/Query.java
@@ -106,8 +106,7 @@ public abstract static class ResultType implements java.io.Serializable {
     private final Class resultClass;
     private final DatastoreV1.EntityResult.ResultType queryType;
 
-    @SuppressWarnings("unchecked")
-    private ResultType(DatastoreV1.EntityResult.ResultType queryType, Class resultClass) {
+    private ResultType(DatastoreV1.EntityResult.ResultType queryType, Class resultClass) {
       this.queryType = queryType;
       this.resultClass = resultClass;
       if (queryType != null) {
@@ -115,7 +114,7 @@ private ResultType(DatastoreV1.EntityResult.ResultType queryType, Class resultCl
       }
     }
 
-    public Class resultClass() {
+    public Class resultClass() {
       return resultClass;
     }
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
index 44360987b573..b23c56a7c395 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResults.java
@@ -20,9 +20,9 @@
 
 /**
  * The result of a Google Cloud Datastore query submission.
- * When result is not typed it is possible to cast it to its appropriate type according to
+ * When the result is not typed it is possible to cast it to its appropriate type according to
  * the {@link #resultClass} value.
- * Results are loaded lazily therefore it is possible to get a {@code DatastoreException}
+ * Results are loaded lazily; therefore it is possible to get a {@code DatastoreException}
  * upon {@link Iterator#hasNext hasNext} or {@link Iterator#next next} calls.
  *
  * @param  the type of the results value.
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
index 8e2f294ed15d..cd3fe9dd776b 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/QueryResultsImpl.java
@@ -76,7 +76,6 @@ private void sendRequest() {
         "Unexpected result type " + actualResultType + " vs " + queryResultType);
   }
 
-  @SuppressWarnings("unchecked")
   @Override
   protected T computeNext() {
     while (!entityResultPbIter.hasNext() && !lastBatch) {
@@ -88,7 +87,9 @@ protected T computeNext() {
     }
     DatastoreV1.EntityResult entityResultPb = entityResultPbIter.next();
     //cursor = entityResultPb.getCursor(); // only available in v1beta3
-    return (T) actualResultType.convert(entityResultPb.getEntity());
+    @SuppressWarnings("unchecked")
+    T result = (T) actualResultType.convert(entityResultPb.getEntity());
+    return result;
   }
 
   @Override
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/RawValue.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/RawValue.java
index 550d27804eba..9d447cf4289b 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/RawValue.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/RawValue.java
@@ -48,7 +48,8 @@ protected void setValue(RawValue from, DatastoreV1.Value.Builder to) {
         }
       };
 
-  static final class Builder extends Value.BaseBuilder {
+  public static final class Builder
+      extends Value.BaseBuilder {
 
     private Builder() {
       super(ValueType.RAW_VALUE);
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
index cb141b6218b0..ce596307d6da 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/StructuredQuery.java
@@ -78,7 +78,8 @@
  * } 
    
  *
  * @param  the type of the result values this query will produce
- * @see Datastore queries
+ * @see Datastore
+ *     queries
  */
 public class StructuredQuery extends Query {
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
index 48568650910d..ae677aa005d6 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/TransactionImpl.java
@@ -38,7 +38,7 @@ static class ResponseImpl implements Transaction.Response {
 
     private final DatastoreV1.CommitResponse response;
 
-    public ResponseImpl(DatastoreV1.CommitResponse response) {
+    ResponseImpl(DatastoreV1.CommitResponse response) {
       this.response = response;
     }
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueBuilder.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueBuilder.java
index 99a44f19366a..f5b5d4c1319b 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueBuilder.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueBuilder.java
@@ -19,7 +19,7 @@
 /**
  * A common interface for Value builders.
  */
-interface ValueBuilder, B extends ValueBuilder> {
+public interface ValueBuilder, B extends ValueBuilder> {
 
   ValueType getValueType();
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueType.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueType.java
index 5b515d6a0901..b09583103a59 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueType.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/datastore/ValueType.java
@@ -97,7 +97,8 @@ public enum ValueType {
   }
 
 
-  , B extends ValueBuilder> ValueType(ValueMarshaller marshaller) {
+  , B extends ValueBuilder>
+      ValueType(ValueMarshaller marshaller) {
     this.marshaller = marshaller;
   }
 
diff --git a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
index f4b7cd39587b..dffcc3f0e16f 100644
--- a/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
+++ b/gcloud-java-datastore/src/main/java/com/google/gcloud/spi/DatastoreRpc.java
@@ -38,7 +38,8 @@ public class DatastoreRpcException extends Exception {
     /**
      * The reason for the exception.
      *
-     * @see Google Cloud Datastore error codes
+     * @see Google
+     *     Cloud Datastore error codes
      */
     public enum Reason {
 
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
index 9fe207a04056..ad8e0cee266a 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseDatastoreBatchWriterTest.java
@@ -16,11 +16,14 @@
 
 package com.google.gcloud.datastore;
 
-import static org.easymock.EasyMock.*;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
 import static org.junit.Assert.assertEquals;
 
 import com.google.api.services.datastore.DatastoreV1;
 import com.google.common.collect.ImmutableList;
+
 import org.easymock.EasyMock;
 import org.junit.After;
 import org.junit.Before;
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
index 02ccd996dc83..5ece01508d3a 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BaseEntityTest.java
@@ -43,14 +43,14 @@ public class BaseEntityTest {
 
   private Builder builder;
 
-  private class Builder extends BaseEntity.Builder {
+  private class Builder extends BaseEntity.Builder {
 
-    @Override public BaseEntity build() {
+    @Override public BaseEntity build() {
 
-      return new BaseEntity(this) {
+      return new BaseEntity(this) {
 
         @Override
-        protected Builder emptyBuilder() {
+        protected BaseEntityTest.Builder emptyBuilder() {
           return new BaseEntityTest.Builder();
         }
       };
@@ -71,7 +71,7 @@ public void setUp() {
 
   @Test
   public void testContains() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertTrue(entity.contains("list1"));
     assertFalse(entity.contains("bla"));
     entity = builder.clear().build();
@@ -80,19 +80,19 @@ public void testContains() throws Exception {
 
   @Test
   public void testGetValue() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(BlobValue.of(BLOB), entity.getValue("blob"));
   }
 
   @Test(expected = DatastoreException.class)
   public void testGetValueNotFound() throws Exception {
-    BaseEntity entity = builder.clear().build();
+    BaseEntity entity = builder.clear().build();
     entity.getValue("blob");
   }
 
   @Test
   public void testIsNull() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertTrue(entity.isNull("null"));
     assertFalse(entity.isNull("blob"));
     entity = builder.setNull("blob").build();
@@ -101,13 +101,13 @@ public void testIsNull() throws Exception {
 
   @Test(expected = DatastoreException.class)
   public void testIsNullNotFound() throws Exception {
-    BaseEntity entity = builder.clear().build();
+    BaseEntity entity = builder.clear().build();
     entity.isNull("null");
   }
 
   @Test
   public void testGetString() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals("hello world", entity.getString("string"));
     assertEquals("bla", entity.getString("stringValue"));
     entity = builder.set("string", "foo").build();
@@ -116,7 +116,7 @@ public void testGetString() throws Exception {
 
   @Test
   public void testGetLong() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(125, entity.getLong("long"));
     entity = builder.set("long", LongValue.of(10)).build();
     assertEquals(10, entity.getLong("long"));
@@ -124,7 +124,7 @@ public void testGetLong() throws Exception {
 
   @Test
   public void testGetDouble() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(1.25, entity.getDouble("double"), 0);
     entity = builder.set("double", DoubleValue.of(10)).build();
     assertEquals(10, entity.getDouble("double"), 0);
@@ -132,7 +132,7 @@ public void testGetDouble() throws Exception {
 
   @Test
   public void testGetBoolean() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertTrue(entity.getBoolean("boolean"));
     entity = builder.set("boolean", BooleanValue.of(false)).build();
     assertFalse(entity.getBoolean("boolean"));
@@ -140,7 +140,7 @@ public void testGetBoolean() throws Exception {
 
   @Test
   public void testGetDateTime() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(DATE_TIME, entity.getDateTime("dateTime"));
     Calendar cal = Calendar.getInstance();
     cal.add(Calendar.DATE, -1);
@@ -151,7 +151,7 @@ public void testGetDateTime() throws Exception {
 
   @Test
   public void testGetKey() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(KEY, entity.getKey("key"));
     Key key = Key.builder(KEY).name("BLA").build();
     entity = builder.set("key", key).build();
@@ -160,7 +160,7 @@ public void testGetKey() throws Exception {
 
   @Test
   public void testGetEntity() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(ENTITY, entity.getEntity("entity"));
     assertEquals(PARTIAL_ENTITY, entity.getEntity("partialEntity"));
     entity = builder.set("entity", EntityValue.of(PARTIAL_ENTITY)).build();
@@ -169,7 +169,7 @@ public void testGetEntity() throws Exception {
 
   @Test
   public void testGetList() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     List> list = entity.getList("list1");
     assertEquals(2, list.size());
     assertEquals(NullValue.of(), list.get(0));
@@ -187,7 +187,7 @@ public void testGetList() throws Exception {
 
   @Test
   public void testGetBlob() throws Exception {
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(BLOB, entity.getBlob("blob"));
     Blob blob = Blob.copyFrom(new byte[] {});
     entity = builder.set("blob", BlobValue.of(blob)).build();
@@ -200,7 +200,7 @@ public void testNames() throws Exception {
         .add("string", "stringValue", "boolean", "double", "long", "list1", "list2", "list3")
         .add("entity", "partialEntity", "null", "dateTime", "blob", "key")
         .build();
-    BaseEntity entity = builder.build();
+    BaseEntity entity = builder.build();
     assertEquals(names, entity.names());
   }
 }
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
index 9cd0b4ce332a..1bb7c3fc476e 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/BlobTest.java
@@ -31,8 +31,8 @@
 
 public class BlobTest {
 
-  private byte[] bytes1 = new byte[10];
-  private byte[] bytes2 = new byte[11];
+  private static final byte[] bytes1 = new byte[10];
+  private static final byte[] bytes2 = new byte[11];
   private Blob blob1;
   private Blob blob2;
 
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
index e4a9b6cbc742..72fd3d13cebe 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/CursorTest.java
@@ -26,8 +26,8 @@
 
 public class CursorTest {
 
-  private byte[] bytes1 = {1, 2, 3, '%', '<', '+'};
-  private byte[] bytes2 = {10, 20, 30};
+  private static final byte[] bytes1 = {1, 2, 3, '%', '<', '+'};
+  private static final byte[] bytes2 = {10, 20, 30};
   private Cursor cursor1;
   private Cursor cursor2;
 
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityTest.java
index bb6d1a6eab73..30bdf16d9397 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/EntityTest.java
@@ -16,7 +16,9 @@
 
 package com.google.gcloud.datastore;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
 
 import org.junit.Test;
 
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
index 7ed9a4e830cf..2d7f5802f247 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java
@@ -140,9 +140,9 @@ public void start() throws IOException, InterruptedException {
     File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME);
     if (!gcdZipFile.exists() || !MD5_CHECKSUM.equals(md5(gcdZipFile))) {
       ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream());
-      FileOutputStream fos = new FileOutputStream(gcdZipFile);
-      fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
-      fos.close();
+      try (FileOutputStream fos = new FileOutputStream(gcdZipFile)) {
+        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+      }
     }
     // unzip the gcd
     try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) {
@@ -209,7 +209,7 @@ private static String md5(File gcdZipFile) throws IOException {
   }
 
   private static boolean isWindows() {
-    return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).indexOf("windows") > -1;
+    return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("windows");
   }
 
   private static void extractFile(ZipInputStream zipIn, File filePath) throws IOException {
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
index 0262fb04b89d..43eec2f02001 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ProjectionEntityTest.java
@@ -27,7 +27,8 @@
 public class ProjectionEntityTest {
 
   private static final Key KEY = Key.builder("ds1", "k1", "n1").build();
-  private static final StringValue STRING_INDEX_VALUE = StringValue.builder("foo").meaning(18).build();
+  private static final StringValue STRING_INDEX_VALUE =
+      StringValue.builder("foo").meaning(18).build();
   private static final BlobValue BLOB_VALUE = BlobValue.of(Blob.copyFrom(new byte[]{1}));
   private static final DateTimeValue DATE_TIME_VALUE = DateTimeValue.of(DateTime.now());
   private static final LongValue LONG_INDEX_VALUE =
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
index 9574f1e246d2..5f3bfc036fa2 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/SerializationTest.java
@@ -57,7 +57,8 @@ public class SerializationTest {
       .namespace("ns1")
       .build();
   private static final Query GQL2 =
-      Query.gqlQueryBuilder(Query.ResultType.ENTITY, "select * from kind1 where name = @name and age > @1")
+      Query.gqlQueryBuilder(
+          Query.ResultType.ENTITY, "select * from kind1 where name = @name and age > @1")
       .setBinding("name", "name1")
       .addBinding(20)
       .namespace("ns1")
@@ -166,10 +167,10 @@ public void testValues() throws Exception {
 
   @Test
   public void testTypes() throws Exception {
-    Serializable[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
+    Serializable[] types = { KEY1, KEY2, INCOMPLETE_KEY1, INCOMPLETE_KEY2, ENTITY1, ENTITY2,
         ENTITY3, EMBEDDED_ENTITY, PROJECTION_ENTITY, DATE_TIME1, BLOB1, CURSOR1, GQL1, GQL2,
         QUERY1, QUERY2, QUERY3};
-    for (Serializable obj : types) {
+    for (Serializable obj : types) {
       Object copy = serializeAndDeserialize(obj);
       assertEquals(obj, obj);
       assertEquals(obj, copy);
@@ -178,7 +179,6 @@ public void testTypes() throws Exception {
     }
   }
 
-  @SuppressWarnings("unchecked")
   private  T serializeAndDeserialize(T obj)
       throws IOException, ClassNotFoundException {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@@ -187,7 +187,9 @@ private  T serializeAndDeserialize(T obj)
     }
     try (ObjectInputStream input =
         new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
-      return (T) input.readObject();
+      @SuppressWarnings("unchecked")
+      T result = (T) input.readObject();
+      return result;
     }
   }
 }
diff --git a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
index bbfb790b69a2..973a3c3c0da4 100644
--- a/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
+++ b/gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/ValueTest.java
@@ -81,6 +81,7 @@ public void setUp() throws Exception {
     ImmutableMap.Builder> builder = ImmutableMap.builder();
     for (ValueType valueType : ValueType.values()) {
       Object[] values = TYPES.get(valueType);
+      @SuppressWarnings("unchecked")
       Class> valueClass = (Class>) values[0];
       Object value = values[1];
       if (value == null) {
diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
index 9188117e4327..c707e6686707 100644
--- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/DatastoreExample.java
@@ -171,9 +171,6 @@ public String getRequiredParams() {
   }
 
   public static void main(String... args) {
-    DatastoreAction action = null;
-    Datastore datastore = null;
-    Key key = null;
     String projectId = args.length > 0 ? args[0] : null;
     // If you want to access a local Datastore running via the gcd sdk, do
     //   DatastoreOptions options = DatastoreOptions.builder()
@@ -186,11 +183,11 @@ public static void main(String... args) {
         .namespace(NAMESPACE)
         .build();
     String name = args.length > 1 ? args[1] : System.getProperty("user.name");
-    datastore = DatastoreFactory.instance().get(options);
+    Datastore datastore = DatastoreFactory.instance().get(options);
     KeyFactory keyFactory = datastore.newKeyFactory().kind(USER_KIND);
-    key = keyFactory.newKey(name);
+    Key key = keyFactory.newKey(name);
     String actionName = args.length > 2 ? args[2].toLowerCase() : DEFAULT_ACTION;
-    action = ACTIONS.get(actionName);
+    DatastoreAction action = ACTIONS.get(actionName);
     if (action == null) {
       StringBuilder actionAndParams = new StringBuilder();
       for (Map.Entry entry : ACTIONS.entrySet()) {
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java
index d77bb1eaef02..e5e319b39164 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Acl.java
@@ -182,7 +182,7 @@ public static final class Project extends Entity {
     private final ProjectRole pRole;
     private final String projectId;
 
-    enum ProjectRole {
+    public enum ProjectRole {
       OWNERS, EDITORS, VIEWERS
     }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
index 45aa1674b03c..02b1ca966622 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BatchResponse.java
@@ -37,7 +37,7 @@ public final class BatchResponse implements Serializable {
   public static class Result implements Serializable {
 
     private static final long serialVersionUID = -1946539570170529094L;
-    private static final Result EMPTY = new BatchResponse.Result(null);
+    private static final Result EMPTY = Result.of(null);
 
     private final T value;
     private final StorageException exception;
@@ -106,9 +106,10 @@ public String toString() {
           .toString();
     }
 
-    @SuppressWarnings("unchecked")
     static  Result empty() {
-      return EMPTY;
+      @SuppressWarnings("unchecked")
+      Result result = (Result) EMPTY;
+      return result;
     }
   }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
index 27cd807f043f..1eefe87702e6 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobWriterChannelImpl.java
@@ -50,7 +50,7 @@ class BlobWriterChannelImpl implements BlobWriteChannel {
   private transient StorageRpc storageRpc;
   private transient StorageObject storageObject;
 
-  public BlobWriterChannelImpl(StorageOptions options, BlobInfo blobInfo,
+  BlobWriterChannelImpl(StorageOptions options, BlobInfo blobInfo,
       Map optionsMap) {
     this.options = options;
     this.blobInfo = blobInfo;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
index 60926e01dbb2..437ce1a142d5 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BucketInfo.java
@@ -46,7 +46,8 @@
 /**
  * A Google Storage bucket.
  *
- * @see Concepts and Terminology
+ * @see Concepts and
+ *     Terminology
  */
 public final class BucketInfo implements Serializable {
 
@@ -169,6 +170,7 @@ public int daysToLive() {
       return daysToLive;
     }
 
+    @Override
     void populateCondition(Rule.Condition condition) {
       condition.setAge(daysToLive);
     }
@@ -185,6 +187,7 @@ static class RawDeleteRule extends DeleteRule {
       this.rule = rule;
     }
 
+    @Override
     void populateCondition(Rule.Condition condition) {
       throw new UnsupportedOperationException();
     }
@@ -219,6 +222,7 @@ public long timeMillis() {
       return timeMillis;
     }
 
+    @Override
     void populateCondition(Rule.Condition condition) {
       condition.setCreatedBefore(new DateTime(timeMillis));
     }
@@ -238,6 +242,7 @@ public int numNewerVersions() {
       return numNewerVersions;
     }
 
+    @Override
     void populateCondition(Rule.Condition condition) {
       condition.setNumNewerVersions(numNewerVersions);
     }
@@ -257,6 +262,7 @@ public boolean isLive() {
       return isLive;
     }
 
+    @Override
     void populateCondition(Rule.Condition condition) {
       condition.setIsLive(isLive);
     }
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java
index f9319f903760..267226c5535f 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/ListResult.java
@@ -33,7 +33,7 @@ public final class ListResult implements Iterable, Se
   private final Iterable results;
   private final NextPageFetcher pageFetcher;
 
-  interface NextPageFetcher extends Serializable {
+  public interface NextPageFetcher extends Serializable {
     ListResult nextPage();
   }
 
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index 6427bfaccf58..e3cfbc195860 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -266,7 +266,8 @@ public static SignUrlOption withMd5() {
      * Service account credentials which are used for signing the URL.
      * If not provided an attempt will be made to get it from the environment.
      *
-     * @see Service account
+     * @see Service
+     *     account
      */
     public static SignUrlOption serviceAccount(ServiceAccountAuthCredentials credentials) {
       return new SignUrlOption(Option.SERVICE_ACCOUNT_CRED, credentials);
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
index 9e4ba2b72407..40b495bc4b47 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java
@@ -104,7 +104,7 @@ public boolean equals(Object obj) {
       return false;
     }
     StorageOptions other = (StorageOptions) obj;
-    return isEquals(other) && Objects.equals(pathDelimiter, other.pathDelimiter);
+    return baseEquals(other) && Objects.equals(pathDelimiter, other.pathDelimiter);
   }
 
   public static StorageOptions defaultInstance() {
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java
index 8a2e69d0c084..c64bc0f98324 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ListResultTest.java
@@ -31,14 +31,14 @@ public void testListResult() throws Exception {
     ImmutableList values = ImmutableList.of("1", "2");
     final ListResult nextResult =
         new ListResult<>(null, "c", Collections.emptyList());
-    ListResult.NextPageFetcher fetcher = new ListResult.NextPageFetcher() {
+    ListResult.NextPageFetcher fetcher = new ListResult.NextPageFetcher() {
 
       @Override
-      public ListResult nextPage() {
+      public ListResult nextPage() {
         return nextResult;
       }
     };
-    ListResult result = new ListResult(fetcher, "c", values);
+    ListResult result = new ListResult<>(fetcher, "c", values);
     assertEquals(nextResult, result.nextPage());
     assertEquals("c", result.nextPageCursor());
     assertEquals(values, ImmutableList.copyOf(result.iterator()));
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
index e79163da3a7e..81342e4b5748 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/SerializationTest.java
@@ -52,17 +52,17 @@ public class SerializationTest {
       Collections.>emptyList());
   private static final ListResult LIST_RESULT =
       new ListResult<>(null, "c", Collections.singletonList(BlobInfo.of("b", "n")));
-  private static Storage.BlobListOption BLOB_LIST_OPTIONS =
+  private static final Storage.BlobListOption BLOB_LIST_OPTIONS =
       Storage.BlobListOption.maxResults(100);
-  private static Storage.BlobSourceOption BLOB_SOURCE_OPTIONS =
+  private static final Storage.BlobSourceOption BLOB_SOURCE_OPTIONS =
       Storage.BlobSourceOption.generationMatch(1);
-  private static Storage.BlobTargetOption BLOB_TARGET_OPTIONS =
+  private static final Storage.BlobTargetOption BLOB_TARGET_OPTIONS =
       Storage.BlobTargetOption.generationMatch();
-  private static Storage.BucketListOption BUCKET_LIST_OPTIONS =
+  private static final Storage.BucketListOption BUCKET_LIST_OPTIONS =
       Storage.BucketListOption.prefix("bla");
-  private static Storage.BucketSourceOption BUCKET_SOURCE_OPTIONS =
+  private static final Storage.BucketSourceOption BUCKET_SOURCE_OPTIONS =
       Storage.BucketSourceOption.metagenerationMatch(1);
-  private static Storage.BucketTargetOption BUCKET_TARGET_OPTIONS =
+  private static final Storage.BucketTargetOption BUCKET_TARGET_OPTIONS =
       Storage.BucketTargetOption.metagenerationNotMatch();
 
   @Test
diff --git a/src/main/java/com/google/gcloud/storage/HttpMethod.java b/src/main/java/com/google/gcloud/storage/HttpMethod.java
deleted file mode 100644
index f5889aedae90..000000000000
--- a/src/main/java/com/google/gcloud/storage/HttpMethod.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2015 Google Inc. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.google.gcloud.storage;
-
-/**
- *
- */
-public enum HttpMethod {
-  GET, HEAD, PUT, POST, DELETE
-}

From 39fb163f53a5713b2c26947ca93f3e732e24b12d Mon Sep 17 00:00:00 2001
From: Eamonn McManus 
Date: Sun, 7 Jun 2015 14:42:13 -0700
Subject: [PATCH 259/732] Adjust type variable usage in BaseService to fix
 compile error.

---
 .../src/main/java/com/google/gcloud/BaseService.java     | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java b/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
index 02bfd76771b2..982d3058295c 100644
--- a/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
+++ b/gcloud-java-core/src/main/java/com/google/gcloud/BaseService.java
@@ -16,16 +16,17 @@
 
 package com.google.gcloud;
 
-public abstract class BaseService implements Service {
+public abstract class BaseService>
+    implements Service {
 
-  private final O options;
+  private final OptionsT options;
 
-  protected BaseService(O options) {
+  protected BaseService(OptionsT options) {
     this.options = options;
   }
 
   @Override
-  public O options() {
+  public OptionsT options() {
     return options;
   }
 }

From 3232488f67b99fe3fa65460ca02a02fac6c69346 Mon Sep 17 00:00:00 2001
From: Stephen Sawchuk 
Date: Mon, 8 Jun 2015 10:41:44 -0400
Subject: [PATCH 260/732] readme: point service links to readme intro
 documentation

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2baf37e639ad..47c6122cf7a8 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ Java idiomatic client for [Google Cloud Platform][cloud-platform] services.
 
 This client supports the following Google Cloud Platform services:
 
--  [Google Cloud Datastore] (https://cloud.google.com/datastore/)
+-  [Google Cloud Datastore] (#google-cloud-datastore)
 
 
+
+
+
+
diff --git a/src/resources/img/icon-dropdown-faq.svg b/src/resources/img/icon-dropdown-faq.svg
new file mode 100644
index 000000000000..786bcdc7d131
--- /dev/null
+++ b/src/resources/img/icon-dropdown-faq.svg
@@ -0,0 +1,7 @@
+
+
+
+
+
+
diff --git a/src/resources/img/icon-dropdown.svg b/src/resources/img/icon-dropdown.svg
new file mode 100644
index 000000000000..3642565ff6b5
--- /dev/null
+++ b/src/resources/img/icon-dropdown.svg
@@ -0,0 +1,7 @@
+
+
+
+
+
+
diff --git a/src/resources/img/icon-lang-java.svg b/src/resources/img/icon-lang-java.svg
new file mode 100644
index 000000000000..58fed060ebd9
--- /dev/null
+++ b/src/resources/img/icon-lang-java.svg
@@ -0,0 +1,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/resources/img/icon-lang-nodejs.svg b/src/resources/img/icon-lang-nodejs.svg
new file mode 100644
index 000000000000..24a4addc3c57
--- /dev/null
+++ b/src/resources/img/icon-lang-nodejs.svg
@@ -0,0 +1,16 @@
+
+
+
+
+
+	
+	
+	
+	
+	
+
+
diff --git a/src/resources/img/icon-lang-python.svg b/src/resources/img/icon-lang-python.svg
new file mode 100644
index 000000000000..bc4737703c35
--- /dev/null
+++ b/src/resources/img/icon-lang-python.svg
@@ -0,0 +1,21 @@
+
+
+
+
+
+	
+		
+		
+	
+
+
diff --git a/src/resources/img/icon-lang-ruby.svg b/src/resources/img/icon-lang-ruby.svg
new file mode 100644
index 000000000000..acfaab8d6ea5
--- /dev/null
+++ b/src/resources/img/icon-lang-ruby.svg
@@ -0,0 +1,23 @@
+
+
+
+
+
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+
+
diff --git a/src/resources/img/icon-link-github.svg b/src/resources/img/icon-link-github.svg
new file mode 100644
index 000000000000..2404f8b0be06
--- /dev/null
+++ b/src/resources/img/icon-link-github.svg
@@ -0,0 +1,19 @@
+
+
+
+
+
+
diff --git a/src/resources/img/icon-link-package-manager.svg b/src/resources/img/icon-link-package-manager.svg
new file mode 100644
index 000000000000..3a12655fe6f9
--- /dev/null
+++ b/src/resources/img/icon-link-package-manager.svg
@@ -0,0 +1,19 @@
+
+
+
+
+
+	
+		
+	
+	
+		
+	
+	
+		
+			
+		
+	
+
+
diff --git a/src/resources/img/icon-link-stackoverflow.svg b/src/resources/img/icon-link-stackoverflow.svg
new file mode 100644
index 000000000000..e1a1f789a897
--- /dev/null
+++ b/src/resources/img/icon-link-stackoverflow.svg
@@ -0,0 +1,16 @@
+
+
+
+
+
+	
+	
+	
+	
+	
+	
+	
+
+
diff --git a/src/resources/img/icon-menu.svg b/src/resources/img/icon-menu.svg
new file mode 100644
index 000000000000..98d3e7073cde
--- /dev/null
+++ b/src/resources/img/icon-menu.svg
@@ -0,0 +1,11 @@
+
+
+
+
+
+	
+	
+	
+
+
diff --git a/src/resources/img/icon-table-check.svg b/src/resources/img/icon-table-check.svg
new file mode 100644
index 000000000000..7934bef97f06
--- /dev/null
+++ b/src/resources/img/icon-table-check.svg
@@ -0,0 +1,7 @@
+
+
+
+
+
+
diff --git a/src/resources/img/lang-bg.png b/src/resources/img/lang-bg.png
new file mode 100644
index 0000000000000000000000000000000000000000..654e12af3418935fe10e84aac1b3134fdc4c4b4b
GIT binary patch
literal 966
zcmaJ=J#W)M7`Dn+s0ggk>EyyhVxR3KPOO+Hu@f2zM^PHoY}BzYiIv)C>`UWhg8_-(
zfY_Ls7%DNqfRNY_5(_&UDwR-o&S{)7kXkz5d#C4l-p_m4ZaqwIZf!CQlWy+l9lD>;
zbt|<&|BwHCzfQM1q~0UDctECRfS9U{`v^2$bBH>~v}aGhpfbZ;0#3I_dd4Hg!Y*gV
z7%p;s%4V2yCGt&cj0otXq2sCS@An^A;Mgj=mp7o{YiQ)`%mcJLZ*{Ht*ecm<F+t=Wc%c%h>_S(ec6`mVU;!axl|2ipXS9Ka0|fFM%v!JjK|$aIDJRQv28d9A
zJQR6B%nGt1i3)^Z`LWbnU=Ngz-dOsgCzTx$;wwBqold!_#NlAb3#C#i))2)kMP$R7
zN6aYeh1-dOjzTMNeBxja#ENDgPl(FWNY7Gm{gtd2F3Us<#z&^l3mlA7N`QuOG1PTe
z=#X^KdA$D>4!biS@f{T6Nnp{&4Yp%fzM=)lBsl0|e2`SJJ;DTsBkTihHxKq4&&Jbm
zfi?`K>4n7fEY#Fhmhw5rv6Ywmp;QG|FjsoWp*^L0ZV3DxmtdB)ltGDFZ>thkua(NmIAGh_NHYZnfc4Bii
zwDhnVd*sns59m-IonHGHKZvGY?M6R-eXFJ!CiMx{PVi|4JgYeOGOzAldGhY^?X4T>
n7gsfF1ak}X^Vj`_dGr29=J55#@$u1{RQ$4<^_Kp&w*UMu#(pT3

literal 0
HcmV?d00001

diff --git a/src/resources/img/logo-full.svg b/src/resources/img/logo-full.svg
new file mode 100644
index 000000000000..3b84037fccc6
--- /dev/null
+++ b/src/resources/img/logo-full.svg
@@ -0,0 +1,112 @@
+
+
+
+
+
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+	
+
+
diff --git a/src/resources/img/logo.svg b/src/resources/img/logo.svg
new file mode 100644
index 000000000000..6c515095c5ad
--- /dev/null
+++ b/src/resources/img/logo.svg
@@ -0,0 +1,25 @@
+
+
+
+
+
+	
+	
+
+
diff --git a/src/resources/index.html b/src/resources/index.html
new file mode 100644
index 000000000000..0553ad29d9da
--- /dev/null
+++ b/src/resources/index.html
@@ -0,0 +1,160 @@
+
+
+
+
+  
+    
+        
+        
+        gcloud
+        
+        
+        
+        
+        
+        
+        
+    
+    
+        
+
+
+        
    
+            
    
+                
    
+                    
    
+                        
    gcloud
    
+                        
    Google Cloud Client Library for Java - an idiomatic, intuitive, and natural way for Java developers to integrate with Google Cloud Platform services, like Cloud Datastore and Cloud Storage.
    
+                    
    
+                    
    
+                        
    Quickstart with Maven: Add this to your pom.xml
    
+                        
    <dependency>
+  <groupId>com.google.gcloud</groupId>
+  <artifactId>gcloud-java</artifactId>
+  <version>0.0.6</version>
+</dependency>
    
+                    
    
+                
    
+            
    
+
+            
    
+                
    
+                    
+                
    
+            
    
+
+            
    
+                
    
+                  
    What is it?
    
+
+                  
    gcloud is a client library for accessing Google
+		    Cloud Platform services that significantly reduces the boilerplate
+		    code you have to write. The library provides high-level API
+		    abstractions so they're easier to understand. It embraces
+		    idioms of Java, works well with the standard library, and
+		    integrates better with your codebase.
+		    All this means you spend more time creating code that matters
+		    to you.
    
+		  
+		  
    gcloud is configured to access Google Cloud Platform
+		    services and authorize (OAuth 2.0) automatically on your behalf.
+		    With a quick install and a private key, you are up and ready
+		    to go. Better yet, if you are running on a Google Compute Engine
+		    instance, the installation is enough!
    
+		  
+                  
    Retrieve Datastore Entries
    
+		  
+                  
    
+		    
    import com.google.gcloud.datastore.Datastore;
+import com.google.gcloud.datastore.DatastoreFactory;
+import com.google.gcloud.datastore.DatastoreOptions;
+import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.Key;
+import com.google.gcloud.datastore.KeyFactory;
+
+DatastoreOptions options = DatastoreOptions.builder().projectId(PROJECT_ID).build();
+Datastore datastore = DatastoreFactory.instance().get(options);
+KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
+Key key = keyFactory.newKey(keyName);
+Entity entity = datastore.get(key);
    
+                
    
+            
    
+
+            
    
+                
    
+                    
    FAQ
    
+		    
    What is the relationship between the gcloud package and the gcloud command-line tool?
    
+		    
    Both the gcloud command-line tool and gcloud package are a part of the Google Cloud SDK: a collection of tools and libraries that enable you to easily create and manage resources on the Google Cloud Platform. The gcloud command-line tool can be used to manage both your development workflow and your Google Cloud Platform resources while the gcloud package is the Google Cloud Client Library for Java.
    
+
+		    
    What is the relationship between gcloud and the Google APIs Java Client?
    
+		    
    The Google APIs Java Client is a client library for using the broad set of Google APIs. gcloud is built specifically for the Google Cloud Platform and is the recommended way to integrate Google Cloud APIs into your Java applications. If your application requires both Google Cloud Platform and other Google APIs, the 2 libraries may be used by your application.
    
+                
    
+            
    
+        
    
+
+        
+        
+        
+        
+    
+
diff --git a/src/resources/index.html~ b/src/resources/index.html~
new file mode 100644
index 000000000000..993f485ab37b
--- /dev/null
+++ b/src/resources/index.html~
@@ -0,0 +1,162 @@
+
+
+
+
+  
+    
+        
+        
+        gcloud
+        
+        
+        
+        
+        
+        
+        
+    
+    
+        
+
+
+        
    
+            
    
+                
    
+                    
    
+                        
    gcloud
    
+                        
    Google Cloud Client Library for Java - an idiomatic, intuitive, and natural way for Java developers to integrate with Google Cloud Platform services, like Cloud Datastore and Cloud Storage.
    
+                    
    
+                    
    
+                        
    Quickstart with Maven: Add this to your pom.xml
    
+                        
    <dependency>
+  <groupId>com.google.gcloud</groupId>
+  <artifactId>gcloud-java</artifactId>
+  <version>0.0.6</version>
+</dependency>
    
+                    
    
+                
    
+            
    
+
+            
    
+                
    
+                    
+                
    
+            
    
+
+            
    
+                
    
+                  
    What is it?
    
+
+                  
    gcloud is a client library for accessing Google
+		    Cloud Platform services that significantly reduces the boilerplate
+		    code you have to write. The library provides high-level API
+		    abstractions so they're easier to understand. It embraces
+		    idioms of Java, works well with the standard library, and
+		    integrates better with your codebase.
+		    All this means you spend more time creating code that matters
+		    to you.
    
+		  
+		  
    gcloud is configured to access Google Cloud Platform
+		    services and authorize (OAuth 2.0) automatically on your behalf.
+		    With a one-line install and a private key, you are up and ready
+		    to go. Better yet, if you are running on a Google Compute Engine
+		    instance, the one-line install is enough!
    
+		  
+                  
    Retrieve Datastore Entries
    
+		  
+                  
    
+		    
    import com.google.gcloud.datastore.Datastore;
+import com.google.gcloud.datastore.DatastoreFactory;
+import com.google.gcloud.datastore.DatastoreOptions;
+import com.google.gcloud.datastore.Entity;
+import com.google.gcloud.datastore.Key;
+import com.google.gcloud.datastore.KeyFactory;
+
+DatastoreOptions options = DatastoreOptions.builder().projectId(PROJECT_ID).build();
+Datastore datastore = DatastoreFactory.instance().get(options);
+KeyFactory keyFactory = datastore.newKeyFactory().kind(KIND);
+Key key = keyFactory.newKey(keyName);
+Entity entity = datastore.get(key);
    
+                
    
+            
    
+
+            
    
+                
    
+                    
    FAQ
    
+		    
    What is the relationship between the gcloud package and the gcloud command-line tool?
    
+		    
    Both the gcloud command-line tool and gcloud package are a part of the Google Cloud SDK: a collection of tools and libraries that enable you to easily create and manage resources on the Google Cloud Platform. The gcloud command-line tool can be used to manage both your development workflow and your Google Cloud Platform resources while the gcloud package is the Google Cloud Client Library for Java.
    
+
+		    
    What is the relationship between gcloud and the Google APIs Java Client?
    
+		    
    The Google APIs Java Client is a client library for using the broad set of Google APIs. gcloud is built specifically for the Google Cloud Platform and is the recommended way to integrate Google Cloud APIs into your Java applications. If your application requires both Google Cloud Platform and other Google APIs, the 2 libraries may be used by your application.
    
+                
    
+            
    
+        
    
+
+        
+        
+        
+        
+    
+
diff --git a/src/resources/js/main.js b/src/resources/js/main.js
new file mode 100644
index 000000000000..119210a6854b
--- /dev/null
+++ b/src/resources/js/main.js
@@ -0,0 +1,7 @@
+$('.nav-current').click(function(){
+    $('.main-nav').toggleClass('open');
+});
+
+$('.faq-btn').click(function(){
+	$(this).toggleClass('open');
+});
\ No newline at end of file
diff --git a/src/resources/js/plugins.js b/src/resources/js/plugins.js
new file mode 100644
index 000000000000..728680b08d18
--- /dev/null
+++ b/src/resources/js/plugins.js
@@ -0,0 +1,24 @@
+// Avoid `console` errors in browsers that lack a console.
+(function() {
+    var method;
+    var noop = function () {};
+    var methods = [
+        'assert', 'clear', 'count', 'debug', 'dir', 'dirxml', 'error',
+        'exception', 'group', 'groupCollapsed', 'groupEnd', 'info', 'log',
+        'markTimeline', 'profile', 'profileEnd', 'table', 'time', 'timeEnd',
+        'timeStamp', 'trace', 'warn'
+    ];
+    var length = methods.length;
+    var console = (window.console = window.console || {});
+
+    while (length--) {
+        method = methods[length];
+
+        // Only stub undefined methods.
+        if (!console[method]) {
+            console[method] = noop;
+        }
+    }
+}());
+
+// Place any jQuery/helper plugins in here.
diff --git a/src/resources/js/vendor/jquery-1.10.2.min.js b/src/resources/js/vendor/jquery-1.10.2.min.js
new file mode 100644
index 000000000000..da4170647dd1
--- /dev/null
+++ b/src/resources/js/vendor/jquery-1.10.2.min.js
@@ -0,0 +1,6 @@
+/*! jQuery v1.10.2 | (c) 2005, 2013 jQuery Foundation, Inc. | jquery.org/license
+//@ sourceMappingURL=jquery-1.10.2.min.map
+*/
+(function(e,t){var n,r,i=typeof t,o=e.location,a=e.document,s=a.documentElement,l=e.jQuery,u=e.$,c={},p=[],f="1.10.2",d=p.concat,h=p.push,g=p.slice,m=p.indexOf,y=c.toString,v=c.hasOwnProperty,b=f.trim,x=function(e,t){return new x.fn.init(e,t,r)},w=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,T=/\S+/g,C=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,N=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,k=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,E=/^[\],:{}\s]*$/,S=/(?:^|:|,)(?:\s*\[)+/g,A=/\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g,j=/"[^"\\\r\n]*"|true|false|null|-?(?:\d+\.|)\d+(?:[eE][+-]?\d+|)/g,D=/^-ms-/,L=/-([\da-z])/gi,H=function(e,t){return t.toUpperCase()},q=function(e){(a.addEventListener||"load"===e.type||"complete"===a.readyState)&&(_(),x.ready())},_=function(){a.addEventListener?(a.removeEventListener("DOMContentLoaded",q,!1),e.removeEventListener("load",q,!1)):(a.detachEvent("onreadystatechange",q),e.detachEvent("onload",q))};x.fn=x.prototype={jquery:f,constructor:x,init:function(e,n,r){var i,o;if(!e)return this;if("string"==typeof e){if(i="<"===e.charAt(0)&&">"===e.charAt(e.length-1)&&e.length>=3?[null,e,null]:N.exec(e),!i||!i[1]&&n)return!n||n.jquery?(n||r).find(e):this.constructor(n).find(e);if(i[1]){if(n=n instanceof x?n[0]:n,x.merge(this,x.parseHTML(i[1],n&&n.nodeType?n.ownerDocument||n:a,!0)),k.test(i[1])&&x.isPlainObject(n))for(i in n)x.isFunction(this[i])?this[i](n[i]):this.attr(i,n[i]);return this}if(o=a.getElementById(i[2]),o&&o.parentNode){if(o.id!==i[2])return r.find(e);this.length=1,this[0]=o}return this.context=a,this.selector=e,this}return e.nodeType?(this.context=this[0]=e,this.length=1,this):x.isFunction(e)?r.ready(e):(e.selector!==t&&(this.selector=e.selector,this.context=e.context),x.makeArray(e,this))},selector:"",length:0,toArray:function(){return g.call(this)},get:function(e){return null==e?this.toArray():0>e?this[this.length+e]:this[e]},pushStack:function(e){var t=x.merge(this.constructor(),e);return t.prevObject=this,t.context=this.context,t},each:function(e,t){return x.each(this,e,t)},ready:function(e){return x.ready.promise().done(e),this},slice:function(){return this.pushStack(g.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(0>e?t:0);return this.pushStack(n>=0&&t>n?[this[n]]:[])},map:function(e){return this.pushStack(x.map(this,function(t,n){return e.call(t,n,t)}))},end:function(){return this.prevObject||this.constructor(null)},push:h,sort:[].sort,splice:[].splice},x.fn.init.prototype=x.fn,x.extend=x.fn.extend=function(){var e,n,r,i,o,a,s=arguments[0]||{},l=1,u=arguments.length,c=!1;for("boolean"==typeof s&&(c=s,s=arguments[1]||{},l=2),"object"==typeof s||x.isFunction(s)||(s={}),u===l&&(s=this,--l);u>l;l++)if(null!=(o=arguments[l]))for(i in o)e=s[i],r=o[i],s!==r&&(c&&r&&(x.isPlainObject(r)||(n=x.isArray(r)))?(n?(n=!1,a=e&&x.isArray(e)?e:[]):a=e&&x.isPlainObject(e)?e:{},s[i]=x.extend(c,a,r)):r!==t&&(s[i]=r));return s},x.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),noConflict:function(t){return e.$===x&&(e.$=u),t&&e.jQuery===x&&(e.jQuery=l),x},isReady:!1,readyWait:1,holdReady:function(e){e?x.readyWait++:x.ready(!0)},ready:function(e){if(e===!0?!--x.readyWait:!x.isReady){if(!a.body)return setTimeout(x.ready);x.isReady=!0,e!==!0&&--x.readyWait>0||(n.resolveWith(a,[x]),x.fn.trigger&&x(a).trigger("ready").off("ready"))}},isFunction:function(e){return"function"===x.type(e)},isArray:Array.isArray||function(e){return"array"===x.type(e)},isWindow:function(e){return null!=e&&e==e.window},isNumeric:function(e){return!isNaN(parseFloat(e))&&isFinite(e)},type:function(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?c[y.call(e)]||"object":typeof e},isPlainObject:function(e){var n;if(!e||"object"!==x.type(e)||e.nodeType||x.isWindow(e))return!1;try{if(e.constructor&&!v.call(e,"constructor")&&!v.call(e.constructor.prototype,"isPrototypeOf"))return!1}catch(r){return!1}if(x.support.ownLast)for(n in e)return v.call(e,n);for(n in e);return n===t||v.call(e,n)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},error:function(e){throw Error(e)},parseHTML:function(e,t,n){if(!e||"string"!=typeof e)return null;"boolean"==typeof t&&(n=t,t=!1),t=t||a;var r=k.exec(e),i=!n&&[];return r?[t.createElement(r[1])]:(r=x.buildFragment([e],t,i),i&&x(i).remove(),x.merge([],r.childNodes))},parseJSON:function(n){return e.JSON&&e.JSON.parse?e.JSON.parse(n):null===n?n:"string"==typeof n&&(n=x.trim(n),n&&E.test(n.replace(A,"@").replace(j,"]").replace(S,"")))?Function("return "+n)():(x.error("Invalid JSON: "+n),t)},parseXML:function(n){var r,i;if(!n||"string"!=typeof n)return null;try{e.DOMParser?(i=new DOMParser,r=i.parseFromString(n,"text/xml")):(r=new ActiveXObject("Microsoft.XMLDOM"),r.async="false",r.loadXML(n))}catch(o){r=t}return r&&r.documentElement&&!r.getElementsByTagName("parsererror").length||x.error("Invalid XML: "+n),r},noop:function(){},globalEval:function(t){t&&x.trim(t)&&(e.execScript||function(t){e.eval.call(e,t)})(t)},camelCase:function(e){return e.replace(D,"ms-").replace(L,H)},nodeName:function(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()},each:function(e,t,n){var r,i=0,o=e.length,a=M(e);if(n){if(a){for(;o>i;i++)if(r=t.apply(e[i],n),r===!1)break}else for(i in e)if(r=t.apply(e[i],n),r===!1)break}else if(a){for(;o>i;i++)if(r=t.call(e[i],i,e[i]),r===!1)break}else for(i in e)if(r=t.call(e[i],i,e[i]),r===!1)break;return e},trim:b&&!b.call("\ufeff\u00a0")?function(e){return null==e?"":b.call(e)}:function(e){return null==e?"":(e+"").replace(C,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(M(Object(e))?x.merge(n,"string"==typeof e?[e]:e):h.call(n,e)),n},inArray:function(e,t,n){var r;if(t){if(m)return m.call(t,e,n);for(r=t.length,n=n?0>n?Math.max(0,r+n):n:0;r>n;n++)if(n in t&&t[n]===e)return n}return-1},merge:function(e,n){var r=n.length,i=e.length,o=0;if("number"==typeof r)for(;r>o;o++)e[i++]=n[o];else while(n[o]!==t)e[i++]=n[o++];return e.length=i,e},grep:function(e,t,n){var r,i=[],o=0,a=e.length;for(n=!!n;a>o;o++)r=!!t(e[o],o),n!==r&&i.push(e[o]);return i},map:function(e,t,n){var r,i=0,o=e.length,a=M(e),s=[];if(a)for(;o>i;i++)r=t(e[i],i,n),null!=r&&(s[s.length]=r);else for(i in e)r=t(e[i],i,n),null!=r&&(s[s.length]=r);return d.apply([],s)},guid:1,proxy:function(e,n){var r,i,o;return"string"==typeof n&&(o=e[n],n=e,e=o),x.isFunction(e)?(r=g.call(arguments,2),i=function(){return e.apply(n||this,r.concat(g.call(arguments)))},i.guid=e.guid=e.guid||x.guid++,i):t},access:function(e,n,r,i,o,a,s){var l=0,u=e.length,c=null==r;if("object"===x.type(r)){o=!0;for(l in r)x.access(e,n,l,r[l],!0,a,s)}else if(i!==t&&(o=!0,x.isFunction(i)||(s=!0),c&&(s?(n.call(e,i),n=null):(c=n,n=function(e,t,n){return c.call(x(e),n)})),n))for(;u>l;l++)n(e[l],r,s?i:i.call(e[l],l,n(e[l],r)));return o?e:c?n.call(e):u?n(e[0],r):a},now:function(){return(new Date).getTime()},swap:function(e,t,n,r){var i,o,a={};for(o in t)a[o]=e.style[o],e.style[o]=t[o];i=n.apply(e,r||[]);for(o in t)e.style[o]=a[o];return i}}),x.ready.promise=function(t){if(!n)if(n=x.Deferred(),"complete"===a.readyState)setTimeout(x.ready);else if(a.addEventListener)a.addEventListener("DOMContentLoaded",q,!1),e.addEventListener("load",q,!1);else{a.attachEvent("onreadystatechange",q),e.attachEvent("onload",q);var r=!1;try{r=null==e.frameElement&&a.documentElement}catch(i){}r&&r.doScroll&&function o(){if(!x.isReady){try{r.doScroll("left")}catch(e){return setTimeout(o,50)}_(),x.ready()}}()}return n.promise(t)},x.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(e,t){c["[object "+t+"]"]=t.toLowerCase()});function M(e){var t=e.length,n=x.type(e);return x.isWindow(e)?!1:1===e.nodeType&&t?!0:"array"===n||"function"!==n&&(0===t||"number"==typeof t&&t>0&&t-1 in e)}r=x(a),function(e,t){var n,r,i,o,a,s,l,u,c,p,f,d,h,g,m,y,v,b="sizzle"+-new Date,w=e.document,T=0,C=0,N=st(),k=st(),E=st(),S=!1,A=function(e,t){return e===t?(S=!0,0):0},j=typeof t,D=1<<31,L={}.hasOwnProperty,H=[],q=H.pop,_=H.push,M=H.push,O=H.slice,F=H.indexOf||function(e){var t=0,n=this.length;for(;n>t;t++)if(this[t]===e)return t;return-1},B="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",P="[\\x20\\t\\r\\n\\f]",R="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",W=R.replace("w","w#"),$="\\["+P+"*("+R+")"+P+"*(?:([*^$|!~]?=)"+P+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+W+")|)|)"+P+"*\\]",I=":("+R+")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|"+$.replace(3,8)+")*)|.*)\\)|)",z=RegExp("^"+P+"+|((?:^|[^\\\\])(?:\\\\.)*)"+P+"+$","g"),X=RegExp("^"+P+"*,"+P+"*"),U=RegExp("^"+P+"*([>+~]|"+P+")"+P+"*"),V=RegExp(P+"*[+~]"),Y=RegExp("="+P+"*([^\\]'\"]*)"+P+"*\\]","g"),J=RegExp(I),G=RegExp("^"+W+"$"),Q={ID:RegExp("^#("+R+")"),CLASS:RegExp("^\\.("+R+")"),TAG:RegExp("^("+R.replace("w","w*")+")"),ATTR:RegExp("^"+$),PSEUDO:RegExp("^"+I),CHILD:RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+P+"*(even|odd|(([+-]|)(\\d*)n|)"+P+"*(?:([+-]|)"+P+"*(\\d+)|))"+P+"*\\)|)","i"),bool:RegExp("^(?:"+B+")$","i"),needsContext:RegExp("^"+P+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+P+"*((?:-\\d)?\\d*)"+P+"*\\)|)(?=[^-]|$)","i")},K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,et=/^(?:input|select|textarea|button)$/i,tt=/^h\d$/i,nt=/'|\\/g,rt=RegExp("\\\\([\\da-f]{1,6}"+P+"?|("+P+")|.)","ig"),it=function(e,t,n){var r="0x"+t-65536;return r!==r||n?t:0>r?String.fromCharCode(r+65536):String.fromCharCode(55296|r>>10,56320|1023&r)};try{M.apply(H=O.call(w.childNodes),w.childNodes),H[w.childNodes.length].nodeType}catch(ot){M={apply:H.length?function(e,t){_.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function at(e,t,n,i){var o,a,s,l,u,c,d,m,y,x;if((t?t.ownerDocument||t:w)!==f&&p(t),t=t||f,n=n||[],!e||"string"!=typeof e)return n;if(1!==(l=t.nodeType)&&9!==l)return[];if(h&&!i){if(o=Z.exec(e))if(s=o[1]){if(9===l){if(a=t.getElementById(s),!a||!a.parentNode)return n;if(a.id===s)return n.push(a),n}else if(t.ownerDocument&&(a=t.ownerDocument.getElementById(s))&&v(t,a)&&a.id===s)return n.push(a),n}else{if(o[2])return M.apply(n,t.getElementsByTagName(e)),n;if((s=o[3])&&r.getElementsByClassName&&t.getElementsByClassName)return M.apply(n,t.getElementsByClassName(s)),n}if(r.qsa&&(!g||!g.test(e))){if(m=d=b,y=t,x=9===l&&e,1===l&&"object"!==t.nodeName.toLowerCase()){c=mt(e),(d=t.getAttribute("id"))?m=d.replace(nt,"\\$&"):t.setAttribute("id",m),m="[id='"+m+"'] ",u=c.length;while(u--)c[u]=m+yt(c[u]);y=V.test(e)&&t.parentNode||t,x=c.join(",")}if(x)try{return M.apply(n,y.querySelectorAll(x)),n}catch(T){}finally{d||t.removeAttribute("id")}}}return kt(e.replace(z,"$1"),t,n,i)}function st(){var e=[];function t(n,r){return e.push(n+=" ")>o.cacheLength&&delete t[e.shift()],t[n]=r}return t}function lt(e){return e[b]=!0,e}function ut(e){var t=f.createElement("div");try{return!!e(t)}catch(n){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function ct(e,t){var n=e.split("|"),r=e.length;while(r--)o.attrHandle[n[r]]=t}function pt(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&(~t.sourceIndex||D)-(~e.sourceIndex||D);if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function ft(e){return function(t){var n=t.nodeName.toLowerCase();return"input"===n&&t.type===e}}function dt(e){return function(t){var n=t.nodeName.toLowerCase();return("input"===n||"button"===n)&&t.type===e}}function ht(e){return lt(function(t){return t=+t,lt(function(n,r){var i,o=e([],n.length,t),a=o.length;while(a--)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}s=at.isXML=function(e){var t=e&&(e.ownerDocument||e).documentElement;return t?"HTML"!==t.nodeName:!1},r=at.support={},p=at.setDocument=function(e){var n=e?e.ownerDocument||e:w,i=n.defaultView;return n!==f&&9===n.nodeType&&n.documentElement?(f=n,d=n.documentElement,h=!s(n),i&&i.attachEvent&&i!==i.top&&i.attachEvent("onbeforeunload",function(){p()}),r.attributes=ut(function(e){return e.className="i",!e.getAttribute("className")}),r.getElementsByTagName=ut(function(e){return e.appendChild(n.createComment("")),!e.getElementsByTagName("*").length}),r.getElementsByClassName=ut(function(e){return e.innerHTML="
    ",e.firstChild.className="i",2===e.getElementsByClassName("i").length}),r.getById=ut(function(e){return d.appendChild(e).id=b,!n.getElementsByName||!n.getElementsByName(b).length}),r.getById?(o.find.ID=function(e,t){if(typeof t.getElementById!==j&&h){var n=t.getElementById(e);return n&&n.parentNode?[n]:[]}},o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){return e.getAttribute("id")===t}}):(delete o.find.ID,o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){var n=typeof e.getAttributeNode!==j&&e.getAttributeNode("id");return n&&n.value===t}}),o.find.TAG=r.getElementsByTagName?function(e,n){return typeof n.getElementsByTagName!==j?n.getElementsByTagName(e):t}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},o.find.CLASS=r.getElementsByClassName&&function(e,n){return typeof n.getElementsByClassName!==j&&h?n.getElementsByClassName(e):t},m=[],g=[],(r.qsa=K.test(n.querySelectorAll))&&(ut(function(e){e.innerHTML="",e.querySelectorAll("[selected]").length||g.push("\\["+P+"*(?:value|"+B+")"),e.querySelectorAll(":checked").length||g.push(":checked")}),ut(function(e){var t=n.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("t",""),e.querySelectorAll("[t^='']").length&&g.push("[*^$]="+P+"*(?:''|\"\")"),e.querySelectorAll(":enabled").length||g.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),g.push(",.*:")})),(r.matchesSelector=K.test(y=d.webkitMatchesSelector||d.mozMatchesSelector||d.oMatchesSelector||d.msMatchesSelector))&&ut(function(e){r.disconnectedMatch=y.call(e,"div"),y.call(e,"[s!='']:x"),m.push("!=",I)}),g=g.length&&RegExp(g.join("|")),m=m.length&&RegExp(m.join("|")),v=K.test(d.contains)||d.compareDocumentPosition?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},A=d.compareDocumentPosition?function(e,t){if(e===t)return S=!0,0;var i=t.compareDocumentPosition&&e.compareDocumentPosition&&e.compareDocumentPosition(t);return i?1&i||!r.sortDetached&&t.compareDocumentPosition(e)===i?e===n||v(w,e)?-1:t===n||v(w,t)?1:c?F.call(c,e)-F.call(c,t):0:4&i?-1:1:e.compareDocumentPosition?-1:1}:function(e,t){var r,i=0,o=e.parentNode,a=t.parentNode,s=[e],l=[t];if(e===t)return S=!0,0;if(!o||!a)return e===n?-1:t===n?1:o?-1:a?1:c?F.call(c,e)-F.call(c,t):0;if(o===a)return pt(e,t);r=e;while(r=r.parentNode)s.unshift(r);r=t;while(r=r.parentNode)l.unshift(r);while(s[i]===l[i])i++;return i?pt(s[i],l[i]):s[i]===w?-1:l[i]===w?1:0},n):f},at.matches=function(e,t){return at(e,null,null,t)},at.matchesSelector=function(e,t){if((e.ownerDocument||e)!==f&&p(e),t=t.replace(Y,"='$1']"),!(!r.matchesSelector||!h||m&&m.test(t)||g&&g.test(t)))try{var n=y.call(e,t);if(n||r.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(i){}return at(t,f,null,[e]).length>0},at.contains=function(e,t){return(e.ownerDocument||e)!==f&&p(e),v(e,t)},at.attr=function(e,n){(e.ownerDocument||e)!==f&&p(e);var i=o.attrHandle[n.toLowerCase()],a=i&&L.call(o.attrHandle,n.toLowerCase())?i(e,n,!h):t;return a===t?r.attributes||!h?e.getAttribute(n):(a=e.getAttributeNode(n))&&a.specified?a.value:null:a},at.error=function(e){throw Error("Syntax error, unrecognized expression: "+e)},at.uniqueSort=function(e){var t,n=[],i=0,o=0;if(S=!r.detectDuplicates,c=!r.sortStable&&e.slice(0),e.sort(A),S){while(t=e[o++])t===e[o]&&(i=n.push(o));while(i--)e.splice(n[i],1)}return e},a=at.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=a(e)}else if(3===i||4===i)return e.nodeValue}else for(;t=e[r];r++)n+=a(t);return n},o=at.selectors={cacheLength:50,createPseudo:lt,match:Q,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(rt,it),e[3]=(e[4]||e[5]||"").replace(rt,it),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||at.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&at.error(e[0]),e},PSEUDO:function(e){var n,r=!e[5]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]&&e[4]!==t?e[2]=e[4]:r&&J.test(r)&&(n=mt(r,!0))&&(n=r.indexOf(")",r.length-n)-r.length)&&(e[0]=e[0].slice(0,n),e[2]=r.slice(0,n)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(rt,it).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=N[e+" "];return t||(t=RegExp("(^|"+P+")"+e+"("+P+"|$)"))&&N(e,function(e){return t.test("string"==typeof e.className&&e.className||typeof e.getAttribute!==j&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=at.attr(r,e);return null==i?"!="===t:t?(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i+" ").indexOf(n)>-1:"|="===t?i===n||i.slice(0,n.length+1)===n+"-":!1):!0}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,l){var u,c,p,f,d,h,g=o!==a?"nextSibling":"previousSibling",m=t.parentNode,y=s&&t.nodeName.toLowerCase(),v=!l&&!s;if(m){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===y:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?m.firstChild:m.lastChild],a&&v){c=m[b]||(m[b]={}),u=c[e]||[],d=u[0]===T&&u[1],f=u[0]===T&&u[2],p=d&&m.childNodes[d];while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if(1===p.nodeType&&++f&&p===t){c[e]=[T,d,f];break}}else if(v&&(u=(t[b]||(t[b]={}))[e])&&u[0]===T)f=u[1];else while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===y:1===p.nodeType)&&++f&&(v&&((p[b]||(p[b]={}))[e]=[T,f]),p===t))break;return f-=i,f===r||0===f%r&&f/r>=0}}},PSEUDO:function(e,t){var n,r=o.pseudos[e]||o.setFilters[e.toLowerCase()]||at.error("unsupported pseudo: "+e);return r[b]?r(t):r.length>1?(n=[e,e,"",t],o.setFilters.hasOwnProperty(e.toLowerCase())?lt(function(e,n){var i,o=r(e,t),a=o.length;while(a--)i=F.call(e,o[a]),e[i]=!(n[i]=o[a])}):function(e){return r(e,0,n)}):r}},pseudos:{not:lt(function(e){var t=[],n=[],r=l(e.replace(z,"$1"));return r[b]?lt(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),!n.pop()}}),has:lt(function(e){return function(t){return at(e,t).length>0}}),contains:lt(function(e){return function(t){return(t.textContent||t.innerText||a(t)).indexOf(e)>-1}}),lang:lt(function(e){return G.test(e||"")||at.error("unsupported lang: "+e),e=e.replace(rt,it).toLowerCase(),function(t){var n;do if(n=h?t.lang:t.getAttribute("xml:lang")||t.getAttribute("lang"))return n=n.toLowerCase(),n===e||0===n.indexOf(e+"-");while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===d},focus:function(e){return e===f.activeElement&&(!f.hasFocus||f.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:function(e){return e.disabled===!1},disabled:function(e){return e.disabled===!0},checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,e.selected===!0},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeName>"@"||3===e.nodeType||4===e.nodeType)return!1;return!0},parent:function(e){return!o.pseudos.empty(e)},header:function(e){return tt.test(e.nodeName)},input:function(e){return et.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||t.toLowerCase()===e.type)},first:ht(function(){return[0]}),last:ht(function(e,t){return[t-1]}),eq:ht(function(e,t,n){return[0>n?n+t:n]}),even:ht(function(e,t){var n=0;for(;t>n;n+=2)e.push(n);return e}),odd:ht(function(e,t){var n=1;for(;t>n;n+=2)e.push(n);return e}),lt:ht(function(e,t,n){var r=0>n?n+t:n;for(;--r>=0;)e.push(r);return e}),gt:ht(function(e,t,n){var r=0>n?n+t:n;for(;t>++r;)e.push(r);return e})}},o.pseudos.nth=o.pseudos.eq;for(n in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})o.pseudos[n]=ft(n);for(n in{submit:!0,reset:!0})o.pseudos[n]=dt(n);function gt(){}gt.prototype=o.filters=o.pseudos,o.setFilters=new gt;function mt(e,t){var n,r,i,a,s,l,u,c=k[e+" "];if(c)return t?0:c.slice(0);s=e,l=[],u=o.preFilter;while(s){(!n||(r=X.exec(s)))&&(r&&(s=s.slice(r[0].length)||s),l.push(i=[])),n=!1,(r=U.exec(s))&&(n=r.shift(),i.push({value:n,type:r[0].replace(z," ")}),s=s.slice(n.length));for(a in o.filter)!(r=Q[a].exec(s))||u[a]&&!(r=u[a](r))||(n=r.shift(),i.push({value:n,type:a,matches:r}),s=s.slice(n.length));if(!n)break}return t?s.length:s?at.error(e):k(e,l).slice(0)}function yt(e){var t=0,n=e.length,r="";for(;n>t;t++)r+=e[t].value;return r}function vt(e,t,n){var r=t.dir,o=n&&"parentNode"===r,a=C++;return t.first?function(t,n,i){while(t=t[r])if(1===t.nodeType||o)return e(t,n,i)}:function(t,n,s){var l,u,c,p=T+" "+a;if(s){while(t=t[r])if((1===t.nodeType||o)&&e(t,n,s))return!0}else while(t=t[r])if(1===t.nodeType||o)if(c=t[b]||(t[b]={}),(u=c[r])&&u[0]===p){if((l=u[1])===!0||l===i)return l===!0}else if(u=c[r]=[p],u[1]=e(t,n,s)||i,u[1]===!0)return!0}}function bt(e){return e.length>1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function xt(e,t,n,r,i){var o,a=[],s=0,l=e.length,u=null!=t;for(;l>s;s++)(o=e[s])&&(!n||n(o,r,i))&&(a.push(o),u&&t.push(s));return a}function wt(e,t,n,r,i,o){return r&&!r[b]&&(r=wt(r)),i&&!i[b]&&(i=wt(i,o)),lt(function(o,a,s,l){var u,c,p,f=[],d=[],h=a.length,g=o||Nt(t||"*",s.nodeType?[s]:s,[]),m=!e||!o&&t?g:xt(g,f,e,s,l),y=n?i||(o?e:h||r)?[]:a:m;if(n&&n(m,y,s,l),r){u=xt(y,d),r(u,[],s,l),c=u.length;while(c--)(p=u[c])&&(y[d[c]]=!(m[d[c]]=p))}if(o){if(i||e){if(i){u=[],c=y.length;while(c--)(p=y[c])&&u.push(m[c]=p);i(null,y=[],u,l)}c=y.length;while(c--)(p=y[c])&&(u=i?F.call(o,p):f[c])>-1&&(o[u]=!(a[u]=p))}}else y=xt(y===a?y.splice(h,y.length):y),i?i(null,a,y,l):M.apply(a,y)})}function Tt(e){var t,n,r,i=e.length,a=o.relative[e[0].type],s=a||o.relative[" "],l=a?1:0,c=vt(function(e){return e===t},s,!0),p=vt(function(e){return F.call(t,e)>-1},s,!0),f=[function(e,n,r){return!a&&(r||n!==u)||((t=n).nodeType?c(e,n,r):p(e,n,r))}];for(;i>l;l++)if(n=o.relative[e[l].type])f=[vt(bt(f),n)];else{if(n=o.filter[e[l].type].apply(null,e[l].matches),n[b]){for(r=++l;i>r;r++)if(o.relative[e[r].type])break;return wt(l>1&&bt(f),l>1&&yt(e.slice(0,l-1).concat({value:" "===e[l-2].type?"*":""})).replace(z,"$1"),n,r>l&&Tt(e.slice(l,r)),i>r&&Tt(e=e.slice(r)),i>r&&yt(e))}f.push(n)}return bt(f)}function Ct(e,t){var n=0,r=t.length>0,a=e.length>0,s=function(s,l,c,p,d){var h,g,m,y=[],v=0,b="0",x=s&&[],w=null!=d,C=u,N=s||a&&o.find.TAG("*",d&&l.parentNode||l),k=T+=null==C?1:Math.random()||.1;for(w&&(u=l!==f&&l,i=n);null!=(h=N[b]);b++){if(a&&h){g=0;while(m=e[g++])if(m(h,l,c)){p.push(h);break}w&&(T=k,i=++n)}r&&((h=!m&&h)&&v--,s&&x.push(h))}if(v+=b,r&&b!==v){g=0;while(m=t[g++])m(x,y,l,c);if(s){if(v>0)while(b--)x[b]||y[b]||(y[b]=q.call(p));y=xt(y)}M.apply(p,y),w&&!s&&y.length>0&&v+t.length>1&&at.uniqueSort(p)}return w&&(T=k,u=C),x};return r?lt(s):s}l=at.compile=function(e,t){var n,r=[],i=[],o=E[e+" "];if(!o){t||(t=mt(e)),n=t.length;while(n--)o=Tt(t[n]),o[b]?r.push(o):i.push(o);o=E(e,Ct(i,r))}return o};function Nt(e,t,n){var r=0,i=t.length;for(;i>r;r++)at(e,t[r],n);return n}function kt(e,t,n,i){var a,s,u,c,p,f=mt(e);if(!i&&1===f.length){if(s=f[0]=f[0].slice(0),s.length>2&&"ID"===(u=s[0]).type&&r.getById&&9===t.nodeType&&h&&o.relative[s[1].type]){if(t=(o.find.ID(u.matches[0].replace(rt,it),t)||[])[0],!t)return n;e=e.slice(s.shift().value.length)}a=Q.needsContext.test(e)?0:s.length;while(a--){if(u=s[a],o.relative[c=u.type])break;if((p=o.find[c])&&(i=p(u.matches[0].replace(rt,it),V.test(s[0].type)&&t.parentNode||t))){if(s.splice(a,1),e=i.length&&yt(s),!e)return M.apply(n,i),n;break}}}return l(e,f)(i,t,!h,n,V.test(e)),n}r.sortStable=b.split("").sort(A).join("")===b,r.detectDuplicates=S,p(),r.sortDetached=ut(function(e){return 1&e.compareDocumentPosition(f.createElement("div"))}),ut(function(e){return e.innerHTML="","#"===e.firstChild.getAttribute("href")})||ct("type|href|height|width",function(e,n,r){return r?t:e.getAttribute(n,"type"===n.toLowerCase()?1:2)}),r.attributes&&ut(function(e){return e.innerHTML="",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||ct("value",function(e,n,r){return r||"input"!==e.nodeName.toLowerCase()?t:e.defaultValue}),ut(function(e){return null==e.getAttribute("disabled")})||ct(B,function(e,n,r){var i;return r?t:(i=e.getAttributeNode(n))&&i.specified?i.value:e[n]===!0?n.toLowerCase():null}),x.find=at,x.expr=at.selectors,x.expr[":"]=x.expr.pseudos,x.unique=at.uniqueSort,x.text=at.getText,x.isXMLDoc=at.isXML,x.contains=at.contains}(e);var O={};function F(e){var t=O[e]={};return x.each(e.match(T)||[],function(e,n){t[n]=!0}),t}x.Callbacks=function(e){e="string"==typeof e?O[e]||F(e):x.extend({},e);var n,r,i,o,a,s,l=[],u=!e.once&&[],c=function(t){for(r=e.memory&&t,i=!0,a=s||0,s=0,o=l.length,n=!0;l&&o>a;a++)if(l[a].apply(t[0],t[1])===!1&&e.stopOnFalse){r=!1;break}n=!1,l&&(u?u.length&&c(u.shift()):r?l=[]:p.disable())},p={add:function(){if(l){var t=l.length;(function i(t){x.each(t,function(t,n){var r=x.type(n);"function"===r?e.unique&&p.has(n)||l.push(n):n&&n.length&&"string"!==r&&i(n)})})(arguments),n?o=l.length:r&&(s=t,c(r))}return this},remove:function(){return l&&x.each(arguments,function(e,t){var r;while((r=x.inArray(t,l,r))>-1)l.splice(r,1),n&&(o>=r&&o--,a>=r&&a--)}),this},has:function(e){return e?x.inArray(e,l)>-1:!(!l||!l.length)},empty:function(){return l=[],o=0,this},disable:function(){return l=u=r=t,this},disabled:function(){return!l},lock:function(){return u=t,r||p.disable(),this},locked:function(){return!u},fireWith:function(e,t){return!l||i&&!u||(t=t||[],t=[e,t.slice?t.slice():t],n?u.push(t):c(t)),this},fire:function(){return p.fireWith(this,arguments),this},fired:function(){return!!i}};return p},x.extend({Deferred:function(e){var t=[["resolve","done",x.Callbacks("once memory"),"resolved"],["reject","fail",x.Callbacks("once memory"),"rejected"],["notify","progress",x.Callbacks("memory")]],n="pending",r={state:function(){return n},always:function(){return i.done(arguments).fail(arguments),this},then:function(){var e=arguments;return x.Deferred(function(n){x.each(t,function(t,o){var a=o[0],s=x.isFunction(e[t])&&e[t];i[o[1]](function(){var e=s&&s.apply(this,arguments);e&&x.isFunction(e.promise)?e.promise().done(n.resolve).fail(n.reject).progress(n.notify):n[a+"With"](this===r?n.promise():this,s?[e]:arguments)})}),e=null}).promise()},promise:function(e){return null!=e?x.extend(e,r):r}},i={};return r.pipe=r.then,x.each(t,function(e,o){var a=o[2],s=o[3];r[o[1]]=a.add,s&&a.add(function(){n=s},t[1^e][2].disable,t[2][2].lock),i[o[0]]=function(){return i[o[0]+"With"](this===i?r:this,arguments),this},i[o[0]+"With"]=a.fireWith}),r.promise(i),e&&e.call(i,i),i},when:function(e){var t=0,n=g.call(arguments),r=n.length,i=1!==r||e&&x.isFunction(e.promise)?r:0,o=1===i?e:x.Deferred(),a=function(e,t,n){return function(r){t[e]=this,n[e]=arguments.length>1?g.call(arguments):r,n===s?o.notifyWith(t,n):--i||o.resolveWith(t,n)}},s,l,u;if(r>1)for(s=Array(r),l=Array(r),u=Array(r);r>t;t++)n[t]&&x.isFunction(n[t].promise)?n[t].promise().done(a(t,u,n)).fail(o.reject).progress(a(t,l,s)):--i;return i||o.resolveWith(u,n),o.promise()}}),x.support=function(t){var n,r,o,s,l,u,c,p,f,d=a.createElement("div");if(d.setAttribute("className","t"),d.innerHTML="  
    a",n=d.getElementsByTagName("*")||[],r=d.getElementsByTagName("a")[0],!r||!r.style||!n.length)return t;s=a.createElement("select"),u=s.appendChild(a.createElement("option")),o=d.getElementsByTagName("input")[0],r.style.cssText="top:1px;float:left;opacity:.5",t.getSetAttribute="t"!==d.className,t.leadingWhitespace=3===d.firstChild.nodeType,t.tbody=!d.getElementsByTagName("tbody").length,t.htmlSerialize=!!d.getElementsByTagName("link").length,t.style=/top/.test(r.getAttribute("style")),t.hrefNormalized="/a"===r.getAttribute("href"),t.opacity=/^0.5/.test(r.style.opacity),t.cssFloat=!!r.style.cssFloat,t.checkOn=!!o.value,t.optSelected=u.selected,t.enctype=!!a.createElement("form").enctype,t.html5Clone="<:nav>"!==a.createElement("nav").cloneNode(!0).outerHTML,t.inlineBlockNeedsLayout=!1,t.shrinkWrapBlocks=!1,t.pixelPosition=!1,t.deleteExpando=!0,t.noCloneEvent=!0,t.reliableMarginRight=!0,t.boxSizingReliable=!0,o.checked=!0,t.noCloneChecked=o.cloneNode(!0).checked,s.disabled=!0,t.optDisabled=!u.disabled;try{delete d.test}catch(h){t.deleteExpando=!1}o=a.createElement("input"),o.setAttribute("value",""),t.input=""===o.getAttribute("value"),o.value="t",o.setAttribute("type","radio"),t.radioValue="t"===o.value,o.setAttribute("checked","t"),o.setAttribute("name","t"),l=a.createDocumentFragment(),l.appendChild(o),t.appendChecked=o.checked,t.checkClone=l.cloneNode(!0).cloneNode(!0).lastChild.checked,d.attachEvent&&(d.attachEvent("onclick",function(){t.noCloneEvent=!1}),d.cloneNode(!0).click());for(f in{submit:!0,change:!0,focusin:!0})d.setAttribute(c="on"+f,"t"),t[f+"Bubbles"]=c in e||d.attributes[c].expando===!1;d.style.backgroundClip="content-box",d.cloneNode(!0).style.backgroundClip="",t.clearCloneStyle="content-box"===d.style.backgroundClip;for(f in x(t))break;return t.ownLast="0"!==f,x(function(){var n,r,o,s="padding:0;margin:0;border:0;display:block;box-sizing:content-box;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;",l=a.getElementsByTagName("body")[0];l&&(n=a.createElement("div"),n.style.cssText="border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px",l.appendChild(n).appendChild(d),d.innerHTML="
    t
    ",o=d.getElementsByTagName("td"),o[0].style.cssText="padding:0;margin:0;border:0;display:none",p=0===o[0].offsetHeight,o[0].style.display="",o[1].style.display="none",t.reliableHiddenOffsets=p&&0===o[0].offsetHeight,d.innerHTML="",d.style.cssText="box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;",x.swap(l,null!=l.style.zoom?{zoom:1}:{},function(){t.boxSizing=4===d.offsetWidth}),e.getComputedStyle&&(t.pixelPosition="1%"!==(e.getComputedStyle(d,null)||{}).top,t.boxSizingReliable="4px"===(e.getComputedStyle(d,null)||{width:"4px"}).width,r=d.appendChild(a.createElement("div")),r.style.cssText=d.style.cssText=s,r.style.marginRight=r.style.width="0",d.style.width="1px",t.reliableMarginRight=!parseFloat((e.getComputedStyle(r,null)||{}).marginRight)),typeof d.style.zoom!==i&&(d.innerHTML="",d.style.cssText=s+"width:1px;padding:1px;display:inline;zoom:1",t.inlineBlockNeedsLayout=3===d.offsetWidth,d.style.display="block",d.innerHTML="
    ",d.firstChild.style.width="5px",t.shrinkWrapBlocks=3!==d.offsetWidth,t.inlineBlockNeedsLayout&&(l.style.zoom=1)),l.removeChild(n),n=d=o=r=null)}),n=s=l=u=r=o=null,t
+}({});var B=/(?:\{[\s\S]*\}|\[[\s\S]*\])$/,P=/([A-Z])/g;function R(e,n,r,i){if(x.acceptData(e)){var o,a,s=x.expando,l=e.nodeType,u=l?x.cache:e,c=l?e[s]:e[s]&&s;if(c&&u[c]&&(i||u[c].data)||r!==t||"string"!=typeof n)return c||(c=l?e[s]=p.pop()||x.guid++:s),u[c]||(u[c]=l?{}:{toJSON:x.noop}),("object"==typeof n||"function"==typeof n)&&(i?u[c]=x.extend(u[c],n):u[c].data=x.extend(u[c].data,n)),a=u[c],i||(a.data||(a.data={}),a=a.data),r!==t&&(a[x.camelCase(n)]=r),"string"==typeof n?(o=a[n],null==o&&(o=a[x.camelCase(n)])):o=a,o}}function W(e,t,n){if(x.acceptData(e)){var r,i,o=e.nodeType,a=o?x.cache:e,s=o?e[x.expando]:x.expando;if(a[s]){if(t&&(r=n?a[s]:a[s].data)){x.isArray(t)?t=t.concat(x.map(t,x.camelCase)):t in r?t=[t]:(t=x.camelCase(t),t=t in r?[t]:t.split(" ")),i=t.length;while(i--)delete r[t[i]];if(n?!I(r):!x.isEmptyObject(r))return}(n||(delete a[s].data,I(a[s])))&&(o?x.cleanData([e],!0):x.support.deleteExpando||a!=a.window?delete a[s]:a[s]=null)}}}x.extend({cache:{},noData:{applet:!0,embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(e){return e=e.nodeType?x.cache[e[x.expando]]:e[x.expando],!!e&&!I(e)},data:function(e,t,n){return R(e,t,n)},removeData:function(e,t){return W(e,t)},_data:function(e,t,n){return R(e,t,n,!0)},_removeData:function(e,t){return W(e,t,!0)},acceptData:function(e){if(e.nodeType&&1!==e.nodeType&&9!==e.nodeType)return!1;var t=e.nodeName&&x.noData[e.nodeName.toLowerCase()];return!t||t!==!0&&e.getAttribute("classid")===t}}),x.fn.extend({data:function(e,n){var r,i,o=null,a=0,s=this[0];if(e===t){if(this.length&&(o=x.data(s),1===s.nodeType&&!x._data(s,"parsedAttrs"))){for(r=s.attributes;r.length>a;a++)i=r[a].name,0===i.indexOf("data-")&&(i=x.camelCase(i.slice(5)),$(s,i,o[i]));x._data(s,"parsedAttrs",!0)}return o}return"object"==typeof e?this.each(function(){x.data(this,e)}):arguments.length>1?this.each(function(){x.data(this,e,n)}):s?$(s,e,x.data(s,e)):null},removeData:function(e){return this.each(function(){x.removeData(this,e)})}});function $(e,n,r){if(r===t&&1===e.nodeType){var i="data-"+n.replace(P,"-$1").toLowerCase();if(r=e.getAttribute(i),"string"==typeof r){try{r="true"===r?!0:"false"===r?!1:"null"===r?null:+r+""===r?+r:B.test(r)?x.parseJSON(r):r}catch(o){}x.data(e,n,r)}else r=t}return r}function I(e){var t;for(t in e)if(("data"!==t||!x.isEmptyObject(e[t]))&&"toJSON"!==t)return!1;return!0}x.extend({queue:function(e,n,r){var i;return e?(n=(n||"fx")+"queue",i=x._data(e,n),r&&(!i||x.isArray(r)?i=x._data(e,n,x.makeArray(r)):i.push(r)),i||[]):t},dequeue:function(e,t){t=t||"fx";var n=x.queue(e,t),r=n.length,i=n.shift(),o=x._queueHooks(e,t),a=function(){x.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return x._data(e,n)||x._data(e,n,{empty:x.Callbacks("once memory").add(function(){x._removeData(e,t+"queue"),x._removeData(e,n)})})}}),x.fn.extend({queue:function(e,n){var r=2;return"string"!=typeof e&&(n=e,e="fx",r--),r>arguments.length?x.queue(this[0],e):n===t?this:this.each(function(){var t=x.queue(this,e,n);x._queueHooks(this,e),"fx"===e&&"inprogress"!==t[0]&&x.dequeue(this,e)})},dequeue:function(e){return this.each(function(){x.dequeue(this,e)})},delay:function(e,t){return e=x.fx?x.fx.speeds[e]||e:e,t=t||"fx",this.queue(t,function(t,n){var r=setTimeout(t,e);n.stop=function(){clearTimeout(r)}})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,n){var r,i=1,o=x.Deferred(),a=this,s=this.length,l=function(){--i||o.resolveWith(a,[a])};"string"!=typeof e&&(n=e,e=t),e=e||"fx";while(s--)r=x._data(a[s],e+"queueHooks"),r&&r.empty&&(i++,r.empty.add(l));return l(),o.promise(n)}});var z,X,U=/[\t\r\n\f]/g,V=/\r/g,Y=/^(?:input|select|textarea|button|object)$/i,J=/^(?:a|area)$/i,G=/^(?:checked|selected)$/i,Q=x.support.getSetAttribute,K=x.support.input;x.fn.extend({attr:function(e,t){return x.access(this,x.attr,e,t,arguments.length>1)},removeAttr:function(e){return this.each(function(){x.removeAttr(this,e)})},prop:function(e,t){return x.access(this,x.prop,e,t,arguments.length>1)},removeProp:function(e){return e=x.propFix[e]||e,this.each(function(){try{this[e]=t,delete this[e]}catch(n){}})},addClass:function(e){var t,n,r,i,o,a=0,s=this.length,l="string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).addClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):" ")){o=0;while(i=t[o++])0>r.indexOf(" "+i+" ")&&(r+=i+" ");n.className=x.trim(r)}return this},removeClass:function(e){var t,n,r,i,o,a=0,s=this.length,l=0===arguments.length||"string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).removeClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):"")){o=0;while(i=t[o++])while(r.indexOf(" "+i+" ")>=0)r=r.replace(" "+i+" "," ");n.className=e?x.trim(r):""}return this},toggleClass:function(e,t){var n=typeof e;return"boolean"==typeof t&&"string"===n?t?this.addClass(e):this.removeClass(e):x.isFunction(e)?this.each(function(n){x(this).toggleClass(e.call(this,n,this.className,t),t)}):this.each(function(){if("string"===n){var t,r=0,o=x(this),a=e.match(T)||[];while(t=a[r++])o.hasClass(t)?o.removeClass(t):o.addClass(t)}else(n===i||"boolean"===n)&&(this.className&&x._data(this,"__className__",this.className),this.className=this.className||e===!1?"":x._data(this,"__className__")||"")})},hasClass:function(e){var t=" "+e+" ",n=0,r=this.length;for(;r>n;n++)if(1===this[n].nodeType&&(" "+this[n].className+" ").replace(U," ").indexOf(t)>=0)return!0;return!1},val:function(e){var n,r,i,o=this[0];{if(arguments.length)return i=x.isFunction(e),this.each(function(n){var o;1===this.nodeType&&(o=i?e.call(this,n,x(this).val()):e,null==o?o="":"number"==typeof o?o+="":x.isArray(o)&&(o=x.map(o,function(e){return null==e?"":e+""})),r=x.valHooks[this.type]||x.valHooks[this.nodeName.toLowerCase()],r&&"set"in r&&r.set(this,o,"value")!==t||(this.value=o))});if(o)return r=x.valHooks[o.type]||x.valHooks[o.nodeName.toLowerCase()],r&&"get"in r&&(n=r.get(o,"value"))!==t?n:(n=o.value,"string"==typeof n?n.replace(V,""):null==n?"":n)}}}),x.extend({valHooks:{option:{get:function(e){var t=x.find.attr(e,"value");return null!=t?t:e.text}},select:{get:function(e){var t,n,r=e.options,i=e.selectedIndex,o="select-one"===e.type||0>i,a=o?null:[],s=o?i+1:r.length,l=0>i?s:o?i:0;for(;s>l;l++)if(n=r[l],!(!n.selected&&l!==i||(x.support.optDisabled?n.disabled:null!==n.getAttribute("disabled"))||n.parentNode.disabled&&x.nodeName(n.parentNode,"optgroup"))){if(t=x(n).val(),o)return t;a.push(t)}return a},set:function(e,t){var n,r,i=e.options,o=x.makeArray(t),a=i.length;while(a--)r=i[a],(r.selected=x.inArray(x(r).val(),o)>=0)&&(n=!0);return n||(e.selectedIndex=-1),o}}},attr:function(e,n,r){var o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return typeof e.getAttribute===i?x.prop(e,n,r):(1===s&&x.isXMLDoc(e)||(n=n.toLowerCase(),o=x.attrHooks[n]||(x.expr.match.bool.test(n)?X:z)),r===t?o&&"get"in o&&null!==(a=o.get(e,n))?a:(a=x.find.attr(e,n),null==a?t:a):null!==r?o&&"set"in o&&(a=o.set(e,r,n))!==t?a:(e.setAttribute(n,r+""),r):(x.removeAttr(e,n),t))},removeAttr:function(e,t){var n,r,i=0,o=t&&t.match(T);if(o&&1===e.nodeType)while(n=o[i++])r=x.propFix[n]||n,x.expr.match.bool.test(n)?K&&Q||!G.test(n)?e[r]=!1:e[x.camelCase("default-"+n)]=e[r]=!1:x.attr(e,n,""),e.removeAttribute(Q?n:r)},attrHooks:{type:{set:function(e,t){if(!x.support.radioValue&&"radio"===t&&x.nodeName(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},propFix:{"for":"htmlFor","class":"className"},prop:function(e,n,r){var i,o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return a=1!==s||!x.isXMLDoc(e),a&&(n=x.propFix[n]||n,o=x.propHooks[n]),r!==t?o&&"set"in o&&(i=o.set(e,r,n))!==t?i:e[n]=r:o&&"get"in o&&null!==(i=o.get(e,n))?i:e[n]},propHooks:{tabIndex:{get:function(e){var t=x.find.attr(e,"tabindex");return t?parseInt(t,10):Y.test(e.nodeName)||J.test(e.nodeName)&&e.href?0:-1}}}}),X={set:function(e,t,n){return t===!1?x.removeAttr(e,n):K&&Q||!G.test(n)?e.setAttribute(!Q&&x.propFix[n]||n,n):e[x.camelCase("default-"+n)]=e[n]=!0,n}},x.each(x.expr.match.bool.source.match(/\w+/g),function(e,n){var r=x.expr.attrHandle[n]||x.find.attr;x.expr.attrHandle[n]=K&&Q||!G.test(n)?function(e,n,i){var o=x.expr.attrHandle[n],a=i?t:(x.expr.attrHandle[n]=t)!=r(e,n,i)?n.toLowerCase():null;return x.expr.attrHandle[n]=o,a}:function(e,n,r){return r?t:e[x.camelCase("default-"+n)]?n.toLowerCase():null}}),K&&Q||(x.attrHooks.value={set:function(e,n,r){return x.nodeName(e,"input")?(e.defaultValue=n,t):z&&z.set(e,n,r)}}),Q||(z={set:function(e,n,r){var i=e.getAttributeNode(r);return i||e.setAttributeNode(i=e.ownerDocument.createAttribute(r)),i.value=n+="","value"===r||n===e.getAttribute(r)?n:t}},x.expr.attrHandle.id=x.expr.attrHandle.name=x.expr.attrHandle.coords=function(e,n,r){var i;return r?t:(i=e.getAttributeNode(n))&&""!==i.value?i.value:null},x.valHooks.button={get:function(e,n){var r=e.getAttributeNode(n);return r&&r.specified?r.value:t},set:z.set},x.attrHooks.contenteditable={set:function(e,t,n){z.set(e,""===t?!1:t,n)}},x.each(["width","height"],function(e,n){x.attrHooks[n]={set:function(e,r){return""===r?(e.setAttribute(n,"auto"),r):t}}})),x.support.hrefNormalized||x.each(["href","src"],function(e,t){x.propHooks[t]={get:function(e){return e.getAttribute(t,4)}}}),x.support.style||(x.attrHooks.style={get:function(e){return e.style.cssText||t},set:function(e,t){return e.style.cssText=t+""}}),x.support.optSelected||(x.propHooks.selected={get:function(e){var t=e.parentNode;return t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex),null}}),x.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){x.propFix[this.toLowerCase()]=this}),x.support.enctype||(x.propFix.enctype="encoding"),x.each(["radio","checkbox"],function(){x.valHooks[this]={set:function(e,n){return x.isArray(n)?e.checked=x.inArray(x(e).val(),n)>=0:t}},x.support.checkOn||(x.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})});var Z=/^(?:input|select|textarea)$/i,et=/^key/,tt=/^(?:mouse|contextmenu)|click/,nt=/^(?:focusinfocus|focusoutblur)$/,rt=/^([^.]*)(?:\.(.+)|)$/;function it(){return!0}function ot(){return!1}function at(){try{return a.activeElement}catch(e){}}x.event={global:{},add:function(e,n,r,o,a){var s,l,u,c,p,f,d,h,g,m,y,v=x._data(e);if(v){r.handler&&(c=r,r=c.handler,a=c.selector),r.guid||(r.guid=x.guid++),(l=v.events)||(l=v.events={}),(f=v.handle)||(f=v.handle=function(e){return typeof x===i||e&&x.event.triggered===e.type?t:x.event.dispatch.apply(f.elem,arguments)},f.elem=e),n=(n||"").match(T)||[""],u=n.length;while(u--)s=rt.exec(n[u])||[],g=y=s[1],m=(s[2]||"").split(".").sort(),g&&(p=x.event.special[g]||{},g=(a?p.delegateType:p.bindType)||g,p=x.event.special[g]||{},d=x.extend({type:g,origType:y,data:o,handler:r,guid:r.guid,selector:a,needsContext:a&&x.expr.match.needsContext.test(a),namespace:m.join(".")},c),(h=l[g])||(h=l[g]=[],h.delegateCount=0,p.setup&&p.setup.call(e,o,m,f)!==!1||(e.addEventListener?e.addEventListener(g,f,!1):e.attachEvent&&e.attachEvent("on"+g,f))),p.add&&(p.add.call(e,d),d.handler.guid||(d.handler.guid=r.guid)),a?h.splice(h.delegateCount++,0,d):h.push(d),x.event.global[g]=!0);e=null}},remove:function(e,t,n,r,i){var o,a,s,l,u,c,p,f,d,h,g,m=x.hasData(e)&&x._data(e);if(m&&(c=m.events)){t=(t||"").match(T)||[""],u=t.length;while(u--)if(s=rt.exec(t[u])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){p=x.event.special[d]||{},d=(r?p.delegateType:p.bindType)||d,f=c[d]||[],s=s[2]&&RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),l=o=f.length;while(o--)a=f[o],!i&&g!==a.origType||n&&n.guid!==a.guid||s&&!s.test(a.namespace)||r&&r!==a.selector&&("**"!==r||!a.selector)||(f.splice(o,1),a.selector&&f.delegateCount--,p.remove&&p.remove.call(e,a));l&&!f.length&&(p.teardown&&p.teardown.call(e,h,m.handle)!==!1||x.removeEvent(e,d,m.handle),delete c[d])}else for(d in c)x.event.remove(e,d+t[u],n,r,!0);x.isEmptyObject(c)&&(delete m.handle,x._removeData(e,"events"))}},trigger:function(n,r,i,o){var s,l,u,c,p,f,d,h=[i||a],g=v.call(n,"type")?n.type:n,m=v.call(n,"namespace")?n.namespace.split("."):[];if(u=f=i=i||a,3!==i.nodeType&&8!==i.nodeType&&!nt.test(g+x.event.triggered)&&(g.indexOf(".")>=0&&(m=g.split("."),g=m.shift(),m.sort()),l=0>g.indexOf(":")&&"on"+g,n=n[x.expando]?n:new x.Event(g,"object"==typeof n&&n),n.isTrigger=o?2:3,n.namespace=m.join("."),n.namespace_re=n.namespace?RegExp("(^|\\.)"+m.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,n.result=t,n.target||(n.target=i),r=null==r?[n]:x.makeArray(r,[n]),p=x.event.special[g]||{},o||!p.trigger||p.trigger.apply(i,r)!==!1)){if(!o&&!p.noBubble&&!x.isWindow(i)){for(c=p.delegateType||g,nt.test(c+g)||(u=u.parentNode);u;u=u.parentNode)h.push(u),f=u;f===(i.ownerDocument||a)&&h.push(f.defaultView||f.parentWindow||e)}d=0;while((u=h[d++])&&!n.isPropagationStopped())n.type=d>1?c:p.bindType||g,s=(x._data(u,"events")||{})[n.type]&&x._data(u,"handle"),s&&s.apply(u,r),s=l&&u[l],s&&x.acceptData(u)&&s.apply&&s.apply(u,r)===!1&&n.preventDefault();if(n.type=g,!o&&!n.isDefaultPrevented()&&(!p._default||p._default.apply(h.pop(),r)===!1)&&x.acceptData(i)&&l&&i[g]&&!x.isWindow(i)){f=i[l],f&&(i[l]=null),x.event.triggered=g;try{i[g]()}catch(y){}x.event.triggered=t,f&&(i[l]=f)}return n.result}},dispatch:function(e){e=x.event.fix(e);var n,r,i,o,a,s=[],l=g.call(arguments),u=(x._data(this,"events")||{})[e.type]||[],c=x.event.special[e.type]||{};if(l[0]=e,e.delegateTarget=this,!c.preDispatch||c.preDispatch.call(this,e)!==!1){s=x.event.handlers.call(this,e,u),n=0;while((o=s[n++])&&!e.isPropagationStopped()){e.currentTarget=o.elem,a=0;while((i=o.handlers[a++])&&!e.isImmediatePropagationStopped())(!e.namespace_re||e.namespace_re.test(i.namespace))&&(e.handleObj=i,e.data=i.data,r=((x.event.special[i.origType]||{}).handle||i.handler).apply(o.elem,l),r!==t&&(e.result=r)===!1&&(e.preventDefault(),e.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,e),e.result}},handlers:function(e,n){var r,i,o,a,s=[],l=n.delegateCount,u=e.target;if(l&&u.nodeType&&(!e.button||"click"!==e.type))for(;u!=this;u=u.parentNode||this)if(1===u.nodeType&&(u.disabled!==!0||"click"!==e.type)){for(o=[],a=0;l>a;a++)i=n[a],r=i.selector+" ",o[r]===t&&(o[r]=i.needsContext?x(r,this).index(u)>=0:x.find(r,this,null,[u]).length),o[r]&&o.push(i);o.length&&s.push({elem:u,handlers:o})}return n.length>l&&s.push({elem:this,handlers:n.slice(l)}),s},fix:function(e){if(e[x.expando])return e;var t,n,r,i=e.type,o=e,s=this.fixHooks[i];s||(this.fixHooks[i]=s=tt.test(i)?this.mouseHooks:et.test(i)?this.keyHooks:{}),r=s.props?this.props.concat(s.props):this.props,e=new x.Event(o),t=r.length;while(t--)n=r[t],e[n]=o[n];return e.target||(e.target=o.srcElement||a),3===e.target.nodeType&&(e.target=e.target.parentNode),e.metaKey=!!e.metaKey,s.filter?s.filter(e,o):e},props:"altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),fixHooks:{},keyHooks:{props:"char charCode key keyCode".split(" "),filter:function(e,t){return null==e.which&&(e.which=null!=t.charCode?t.charCode:t.keyCode),e}},mouseHooks:{props:"button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),filter:function(e,n){var r,i,o,s=n.button,l=n.fromElement;return null==e.pageX&&null!=n.clientX&&(i=e.target.ownerDocument||a,o=i.documentElement,r=i.body,e.pageX=n.clientX+(o&&o.scrollLeft||r&&r.scrollLeft||0)-(o&&o.clientLeft||r&&r.clientLeft||0),e.pageY=n.clientY+(o&&o.scrollTop||r&&r.scrollTop||0)-(o&&o.clientTop||r&&r.clientTop||0)),!e.relatedTarget&&l&&(e.relatedTarget=l===e.target?n.toElement:l),e.which||s===t||(e.which=1&s?1:2&s?3:4&s?2:0),e}},special:{load:{noBubble:!0},focus:{trigger:function(){if(this!==at()&&this.focus)try{return this.focus(),!1}catch(e){}},delegateType:"focusin"},blur:{trigger:function(){return this===at()&&this.blur?(this.blur(),!1):t},delegateType:"focusout"},click:{trigger:function(){return x.nodeName(this,"input")&&"checkbox"===this.type&&this.click?(this.click(),!1):t},_default:function(e){return x.nodeName(e.target,"a")}},beforeunload:{postDispatch:function(e){e.result!==t&&(e.originalEvent.returnValue=e.result)}}},simulate:function(e,t,n,r){var i=x.extend(new x.Event,n,{type:e,isSimulated:!0,originalEvent:{}});r?x.event.trigger(i,null,t):x.event.dispatch.call(t,i),i.isDefaultPrevented()&&n.preventDefault()}},x.removeEvent=a.removeEventListener?function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n,!1)}:function(e,t,n){var r="on"+t;e.detachEvent&&(typeof e[r]===i&&(e[r]=null),e.detachEvent(r,n))},x.Event=function(e,n){return this instanceof x.Event?(e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||e.returnValue===!1||e.getPreventDefault&&e.getPreventDefault()?it:ot):this.type=e,n&&x.extend(this,n),this.timeStamp=e&&e.timeStamp||x.now(),this[x.expando]=!0,t):new x.Event(e,n)},x.Event.prototype={isDefaultPrevented:ot,isPropagationStopped:ot,isImmediatePropagationStopped:ot,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=it,e&&(e.preventDefault?e.preventDefault():e.returnValue=!1)},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=it,e&&(e.stopPropagation&&e.stopPropagation(),e.cancelBubble=!0)},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=it,this.stopPropagation()}},x.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(e,t){x.event.special[e]={delegateType:t,bindType:t,handle:function(e){var n,r=this,i=e.relatedTarget,o=e.handleObj;return(!i||i!==r&&!x.contains(r,i))&&(e.type=o.origType,n=o.handler.apply(this,arguments),e.type=t),n}}}),x.support.submitBubbles||(x.event.special.submit={setup:function(){return x.nodeName(this,"form")?!1:(x.event.add(this,"click._submit keypress._submit",function(e){var n=e.target,r=x.nodeName(n,"input")||x.nodeName(n,"button")?n.form:t;r&&!x._data(r,"submitBubbles")&&(x.event.add(r,"submit._submit",function(e){e._submit_bubble=!0}),x._data(r,"submitBubbles",!0))}),t)},postDispatch:function(e){e._submit_bubble&&(delete e._submit_bubble,this.parentNode&&!e.isTrigger&&x.event.simulate("submit",this.parentNode,e,!0))},teardown:function(){return x.nodeName(this,"form")?!1:(x.event.remove(this,"._submit"),t)}}),x.support.changeBubbles||(x.event.special.change={setup:function(){return Z.test(this.nodeName)?(("checkbox"===this.type||"radio"===this.type)&&(x.event.add(this,"propertychange._change",function(e){"checked"===e.originalEvent.propertyName&&(this._just_changed=!0)}),x.event.add(this,"click._change",function(e){this._just_changed&&!e.isTrigger&&(this._just_changed=!1),x.event.simulate("change",this,e,!0)})),!1):(x.event.add(this,"beforeactivate._change",function(e){var t=e.target;Z.test(t.nodeName)&&!x._data(t,"changeBubbles")&&(x.event.add(t,"change._change",function(e){!this.parentNode||e.isSimulated||e.isTrigger||x.event.simulate("change",this.parentNode,e,!0)}),x._data(t,"changeBubbles",!0))}),t)},handle:function(e){var n=e.target;return this!==n||e.isSimulated||e.isTrigger||"radio"!==n.type&&"checkbox"!==n.type?e.handleObj.handler.apply(this,arguments):t},teardown:function(){return x.event.remove(this,"._change"),!Z.test(this.nodeName)}}),x.support.focusinBubbles||x.each({focus:"focusin",blur:"focusout"},function(e,t){var n=0,r=function(e){x.event.simulate(t,e.target,x.event.fix(e),!0)};x.event.special[t]={setup:function(){0===n++&&a.addEventListener(e,r,!0)},teardown:function(){0===--n&&a.removeEventListener(e,r,!0)}}}),x.fn.extend({on:function(e,n,r,i,o){var a,s;if("object"==typeof e){"string"!=typeof n&&(r=r||n,n=t);for(a in e)this.on(a,n,r,e[a],o);return this}if(null==r&&null==i?(i=n,r=n=t):null==i&&("string"==typeof n?(i=r,r=t):(i=r,r=n,n=t)),i===!1)i=ot;else if(!i)return this;return 1===o&&(s=i,i=function(e){return x().off(e),s.apply(this,arguments)},i.guid=s.guid||(s.guid=x.guid++)),this.each(function(){x.event.add(this,e,i,r,n)})},one:function(e,t,n,r){return this.on(e,t,n,r,1)},off:function(e,n,r){var i,o;if(e&&e.preventDefault&&e.handleObj)return i=e.handleObj,x(e.delegateTarget).off(i.namespace?i.origType+"."+i.namespace:i.origType,i.selector,i.handler),this;if("object"==typeof e){for(o in e)this.off(o,n,e[o]);return this}return(n===!1||"function"==typeof n)&&(r=n,n=t),r===!1&&(r=ot),this.each(function(){x.event.remove(this,e,r,n)})},trigger:function(e,t){return this.each(function(){x.event.trigger(e,t,this)})},triggerHandler:function(e,n){var r=this[0];return r?x.event.trigger(e,n,r,!0):t}});var st=/^.[^:#\[\.,]*$/,lt=/^(?:parents|prev(?:Until|All))/,ut=x.expr.match.needsContext,ct={children:!0,contents:!0,next:!0,prev:!0};x.fn.extend({find:function(e){var t,n=[],r=this,i=r.length;if("string"!=typeof e)return this.pushStack(x(e).filter(function(){for(t=0;i>t;t++)if(x.contains(r[t],this))return!0}));for(t=0;i>t;t++)x.find(e,r[t],n);return n=this.pushStack(i>1?x.unique(n):n),n.selector=this.selector?this.selector+" "+e:e,n},has:function(e){var t,n=x(e,this),r=n.length;return this.filter(function(){for(t=0;r>t;t++)if(x.contains(this,n[t]))return!0})},not:function(e){return this.pushStack(ft(this,e||[],!0))},filter:function(e){return this.pushStack(ft(this,e||[],!1))},is:function(e){return!!ft(this,"string"==typeof e&&ut.test(e)?x(e):e||[],!1).length},closest:function(e,t){var n,r=0,i=this.length,o=[],a=ut.test(e)||"string"!=typeof e?x(e,t||this.context):0;for(;i>r;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(11>n.nodeType&&(a?a.index(n)>-1:1===n.nodeType&&x.find.matchesSelector(n,e))){n=o.push(n);break}return this.pushStack(o.length>1?x.unique(o):o)},index:function(e){return e?"string"==typeof e?x.inArray(this[0],x(e)):x.inArray(e.jquery?e[0]:e,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){var n="string"==typeof e?x(e,t):x.makeArray(e&&e.nodeType?[e]:e),r=x.merge(this.get(),n);return this.pushStack(x.unique(r))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}});function pt(e,t){do e=e[t];while(e&&1!==e.nodeType);return e}x.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return x.dir(e,"parentNode")},parentsUntil:function(e,t,n){return x.dir(e,"parentNode",n)},next:function(e){return pt(e,"nextSibling")},prev:function(e){return pt(e,"previousSibling")},nextAll:function(e){return x.dir(e,"nextSibling")},prevAll:function(e){return x.dir(e,"previousSibling")},nextUntil:function(e,t,n){return x.dir(e,"nextSibling",n)},prevUntil:function(e,t,n){return x.dir(e,"previousSibling",n)},siblings:function(e){return x.sibling((e.parentNode||{}).firstChild,e)},children:function(e){return x.sibling(e.firstChild)},contents:function(e){return x.nodeName(e,"iframe")?e.contentDocument||e.contentWindow.document:x.merge([],e.childNodes)}},function(e,t){x.fn[e]=function(n,r){var i=x.map(this,t,n);return"Until"!==e.slice(-5)&&(r=n),r&&"string"==typeof r&&(i=x.filter(r,i)),this.length>1&&(ct[e]||(i=x.unique(i)),lt.test(e)&&(i=i.reverse())),this.pushStack(i)}}),x.extend({filter:function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?x.find.matchesSelector(r,e)?[r]:[]:x.find.matches(e,x.grep(t,function(e){return 1===e.nodeType}))},dir:function(e,n,r){var i=[],o=e[n];while(o&&9!==o.nodeType&&(r===t||1!==o.nodeType||!x(o).is(r)))1===o.nodeType&&i.push(o),o=o[n];return i},sibling:function(e,t){var n=[];for(;e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n}});function ft(e,t,n){if(x.isFunction(t))return x.grep(e,function(e,r){return!!t.call(e,r,e)!==n});if(t.nodeType)return x.grep(e,function(e){return e===t!==n});if("string"==typeof t){if(st.test(t))return x.filter(t,e,n);t=x.filter(t,e)}return x.grep(e,function(e){return x.inArray(e,t)>=0!==n})}function dt(e){var t=ht.split("|"),n=e.createDocumentFragment();if(n.createElement)while(t.length)n.createElement(t.pop());return n}var ht="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",gt=/ jQuery\d+="(?:null|\d+)"/g,mt=RegExp("<(?:"+ht+")[\\s/>]","i"),yt=/^\s+/,vt=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,bt=/<([\w:]+)/,xt=/\s*$/g,At={option:[1,""],legend:[1,"
    ","
    "],area:[1,"",""],param:[1,"",""],thead:[1,"","
    "],tr:[2,"","
    "],col:[2,"","
    "],td:[3,"","
    "],_default:x.support.htmlSerialize?[0,"",""]:[1,"X
    ","
    "]},jt=dt(a),Dt=jt.appendChild(a.createElement("div"));At.optgroup=At.option,At.tbody=At.tfoot=At.colgroup=At.caption=At.thead,At.th=At.td,x.fn.extend({text:function(e){return x.access(this,function(e){return e===t?x.text(this):this.empty().append((this[0]&&this[0].ownerDocument||a).createTextNode(e))},null,e,arguments.length)},append:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.appendChild(e)}})},prepend:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},remove:function(e,t){var n,r=e?x.filter(e,this):this,i=0;for(;null!=(n=r[i]);i++)t||1!==n.nodeType||x.cleanData(Ft(n)),n.parentNode&&(t&&x.contains(n.ownerDocument,n)&&_t(Ft(n,"script")),n.parentNode.removeChild(n));return this},empty:function(){var e,t=0;for(;null!=(e=this[t]);t++){1===e.nodeType&&x.cleanData(Ft(e,!1));while(e.firstChild)e.removeChild(e.firstChild);e.options&&x.nodeName(e,"select")&&(e.options.length=0)}return this},clone:function(e,t){return e=null==e?!1:e,t=null==t?e:t,this.map(function(){return x.clone(this,e,t)})},html:function(e){return x.access(this,function(e){var n=this[0]||{},r=0,i=this.length;if(e===t)return 1===n.nodeType?n.innerHTML.replace(gt,""):t;if(!("string"!=typeof e||Tt.test(e)||!x.support.htmlSerialize&&mt.test(e)||!x.support.leadingWhitespace&&yt.test(e)||At[(bt.exec(e)||["",""])[1].toLowerCase()])){e=e.replace(vt,"<$1>");try{for(;i>r;r++)n=this[r]||{},1===n.nodeType&&(x.cleanData(Ft(n,!1)),n.innerHTML=e);n=0}catch(o){}}n&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var e=x.map(this,function(e){return[e.nextSibling,e.parentNode]}),t=0;return this.domManip(arguments,function(n){var r=e[t++],i=e[t++];i&&(r&&r.parentNode!==i&&(r=this.nextSibling),x(this).remove(),i.insertBefore(n,r))},!0),t?this:this.remove()},detach:function(e){return this.remove(e,!0)},domManip:function(e,t,n){e=d.apply([],e);var r,i,o,a,s,l,u=0,c=this.length,p=this,f=c-1,h=e[0],g=x.isFunction(h);if(g||!(1>=c||"string"!=typeof h||x.support.checkClone)&&Nt.test(h))return this.each(function(r){var i=p.eq(r);g&&(e[0]=h.call(this,r,i.html())),i.domManip(e,t,n)});if(c&&(l=x.buildFragment(e,this[0].ownerDocument,!1,!n&&this),r=l.firstChild,1===l.childNodes.length&&(l=r),r)){for(a=x.map(Ft(l,"script"),Ht),o=a.length;c>u;u++)i=l,u!==f&&(i=x.clone(i,!0,!0),o&&x.merge(a,Ft(i,"script"))),t.call(this[u],i,u);if(o)for(s=a[a.length-1].ownerDocument,x.map(a,qt),u=0;o>u;u++)i=a[u],kt.test(i.type||"")&&!x._data(i,"globalEval")&&x.contains(s,i)&&(i.src?x._evalUrl(i.src):x.globalEval((i.text||i.textContent||i.innerHTML||"").replace(St,"")));l=r=null}return this}});function Lt(e,t){return x.nodeName(e,"table")&&x.nodeName(1===t.nodeType?t:t.firstChild,"tr")?e.getElementsByTagName("tbody")[0]||e.appendChild(e.ownerDocument.createElement("tbody")):e}function Ht(e){return e.type=(null!==x.find.attr(e,"type"))+"/"+e.type,e}function qt(e){var t=Et.exec(e.type);return t?e.type=t[1]:e.removeAttribute("type"),e}function _t(e,t){var n,r=0;for(;null!=(n=e[r]);r++)x._data(n,"globalEval",!t||x._data(t[r],"globalEval"))}function Mt(e,t){if(1===t.nodeType&&x.hasData(e)){var n,r,i,o=x._data(e),a=x._data(t,o),s=o.events;if(s){delete a.handle,a.events={};for(n in s)for(r=0,i=s[n].length;i>r;r++)x.event.add(t,n,s[n][r])}a.data&&(a.data=x.extend({},a.data))}}function Ot(e,t){var n,r,i;if(1===t.nodeType){if(n=t.nodeName.toLowerCase(),!x.support.noCloneEvent&&t[x.expando]){i=x._data(t);for(r in i.events)x.removeEvent(t,r,i.handle);t.removeAttribute(x.expando)}"script"===n&&t.text!==e.text?(Ht(t).text=e.text,qt(t)):"object"===n?(t.parentNode&&(t.outerHTML=e.outerHTML),x.support.html5Clone&&e.innerHTML&&!x.trim(t.innerHTML)&&(t.innerHTML=e.innerHTML)):"input"===n&&Ct.test(e.type)?(t.defaultChecked=t.checked=e.checked,t.value!==e.value&&(t.value=e.value)):"option"===n?t.defaultSelected=t.selected=e.defaultSelected:("input"===n||"textarea"===n)&&(t.defaultValue=e.defaultValue)}}x.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,t){x.fn[e]=function(e){var n,r=0,i=[],o=x(e),a=o.length-1;for(;a>=r;r++)n=r===a?this:this.clone(!0),x(o[r])[t](n),h.apply(i,n.get());return this.pushStack(i)}});function Ft(e,n){var r,o,a=0,s=typeof e.getElementsByTagName!==i?e.getElementsByTagName(n||"*"):typeof e.querySelectorAll!==i?e.querySelectorAll(n||"*"):t;if(!s)for(s=[],r=e.childNodes||e;null!=(o=r[a]);a++)!n||x.nodeName(o,n)?s.push(o):x.merge(s,Ft(o,n));return n===t||n&&x.nodeName(e,n)?x.merge([e],s):s}function Bt(e){Ct.test(e.type)&&(e.defaultChecked=e.checked)}x.extend({clone:function(e,t,n){var r,i,o,a,s,l=x.contains(e.ownerDocument,e);if(x.support.html5Clone||x.isXMLDoc(e)||!mt.test("<"+e.nodeName+">")?o=e.cloneNode(!0):(Dt.innerHTML=e.outerHTML,Dt.removeChild(o=Dt.firstChild)),!(x.support.noCloneEvent&&x.support.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||x.isXMLDoc(e)))for(r=Ft(o),s=Ft(e),a=0;null!=(i=s[a]);++a)r[a]&&Ot(i,r[a]);if(t)if(n)for(s=s||Ft(e),r=r||Ft(o),a=0;null!=(i=s[a]);a++)Mt(i,r[a]);else Mt(e,o);return r=Ft(o,"script"),r.length>0&&_t(r,!l&&Ft(e,"script")),r=s=i=null,o},buildFragment:function(e,t,n,r){var i,o,a,s,l,u,c,p=e.length,f=dt(t),d=[],h=0;for(;p>h;h++)if(o=e[h],o||0===o)if("object"===x.type(o))x.merge(d,o.nodeType?[o]:o);else if(wt.test(o)){s=s||f.appendChild(t.createElement("div")),l=(bt.exec(o)||["",""])[1].toLowerCase(),c=At[l]||At._default,s.innerHTML=c[1]+o.replace(vt,"<$1>")+c[2],i=c[0];while(i--)s=s.lastChild;if(!x.support.leadingWhitespace&&yt.test(o)&&d.push(t.createTextNode(yt.exec(o)[0])),!x.support.tbody){o="table"!==l||xt.test(o)?""!==c[1]||xt.test(o)?0:s:s.firstChild,i=o&&o.childNodes.length;while(i--)x.nodeName(u=o.childNodes[i],"tbody")&&!u.childNodes.length&&o.removeChild(u)}x.merge(d,s.childNodes),s.textContent="";while(s.firstChild)s.removeChild(s.firstChild);s=f.lastChild}else d.push(t.createTextNode(o));s&&f.removeChild(s),x.support.appendChecked||x.grep(Ft(d,"input"),Bt),h=0;while(o=d[h++])if((!r||-1===x.inArray(o,r))&&(a=x.contains(o.ownerDocument,o),s=Ft(f.appendChild(o),"script"),a&&_t(s),n)){i=0;while(o=s[i++])kt.test(o.type||"")&&n.push(o)}return s=null,f},cleanData:function(e,t){var n,r,o,a,s=0,l=x.expando,u=x.cache,c=x.support.deleteExpando,f=x.event.special;for(;null!=(n=e[s]);s++)if((t||x.acceptData(n))&&(o=n[l],a=o&&u[o])){if(a.events)for(r in a.events)f[r]?x.event.remove(n,r):x.removeEvent(n,r,a.handle);
+u[o]&&(delete u[o],c?delete n[l]:typeof n.removeAttribute!==i?n.removeAttribute(l):n[l]=null,p.push(o))}},_evalUrl:function(e){return x.ajax({url:e,type:"GET",dataType:"script",async:!1,global:!1,"throws":!0})}}),x.fn.extend({wrapAll:function(e){if(x.isFunction(e))return this.each(function(t){x(this).wrapAll(e.call(this,t))});if(this[0]){var t=x(e,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstChild&&1===e.firstChild.nodeType)e=e.firstChild;return e}).append(this)}return this},wrapInner:function(e){return x.isFunction(e)?this.each(function(t){x(this).wrapInner(e.call(this,t))}):this.each(function(){var t=x(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=x.isFunction(e);return this.each(function(n){x(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(){return this.parent().each(function(){x.nodeName(this,"body")||x(this).replaceWith(this.childNodes)}).end()}});var Pt,Rt,Wt,$t=/alpha\([^)]*\)/i,It=/opacity\s*=\s*([^)]*)/,zt=/^(top|right|bottom|left)$/,Xt=/^(none|table(?!-c[ea]).+)/,Ut=/^margin/,Vt=RegExp("^("+w+")(.*)$","i"),Yt=RegExp("^("+w+")(?!px)[a-z%]+$","i"),Jt=RegExp("^([+-])=("+w+")","i"),Gt={BODY:"block"},Qt={position:"absolute",visibility:"hidden",display:"block"},Kt={letterSpacing:0,fontWeight:400},Zt=["Top","Right","Bottom","Left"],en=["Webkit","O","Moz","ms"];function tn(e,t){if(t in e)return t;var n=t.charAt(0).toUpperCase()+t.slice(1),r=t,i=en.length;while(i--)if(t=en[i]+n,t in e)return t;return r}function nn(e,t){return e=t||e,"none"===x.css(e,"display")||!x.contains(e.ownerDocument,e)}function rn(e,t){var n,r,i,o=[],a=0,s=e.length;for(;s>a;a++)r=e[a],r.style&&(o[a]=x._data(r,"olddisplay"),n=r.style.display,t?(o[a]||"none"!==n||(r.style.display=""),""===r.style.display&&nn(r)&&(o[a]=x._data(r,"olddisplay",ln(r.nodeName)))):o[a]||(i=nn(r),(n&&"none"!==n||!i)&&x._data(r,"olddisplay",i?n:x.css(r,"display"))));for(a=0;s>a;a++)r=e[a],r.style&&(t&&"none"!==r.style.display&&""!==r.style.display||(r.style.display=t?o[a]||"":"none"));return e}x.fn.extend({css:function(e,n){return x.access(this,function(e,n,r){var i,o,a={},s=0;if(x.isArray(n)){for(o=Rt(e),i=n.length;i>s;s++)a[n[s]]=x.css(e,n[s],!1,o);return a}return r!==t?x.style(e,n,r):x.css(e,n)},e,n,arguments.length>1)},show:function(){return rn(this,!0)},hide:function(){return rn(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){nn(this)?x(this).show():x(this).hide()})}}),x.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Wt(e,"opacity");return""===n?"1":n}}}},cssNumber:{columnCount:!0,fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":x.support.cssFloat?"cssFloat":"styleFloat"},style:function(e,n,r,i){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var o,a,s,l=x.camelCase(n),u=e.style;if(n=x.cssProps[l]||(x.cssProps[l]=tn(u,l)),s=x.cssHooks[n]||x.cssHooks[l],r===t)return s&&"get"in s&&(o=s.get(e,!1,i))!==t?o:u[n];if(a=typeof r,"string"===a&&(o=Jt.exec(r))&&(r=(o[1]+1)*o[2]+parseFloat(x.css(e,n)),a="number"),!(null==r||"number"===a&&isNaN(r)||("number"!==a||x.cssNumber[l]||(r+="px"),x.support.clearCloneStyle||""!==r||0!==n.indexOf("background")||(u[n]="inherit"),s&&"set"in s&&(r=s.set(e,r,i))===t)))try{u[n]=r}catch(c){}}},css:function(e,n,r,i){var o,a,s,l=x.camelCase(n);return n=x.cssProps[l]||(x.cssProps[l]=tn(e.style,l)),s=x.cssHooks[n]||x.cssHooks[l],s&&"get"in s&&(a=s.get(e,!0,r)),a===t&&(a=Wt(e,n,i)),"normal"===a&&n in Kt&&(a=Kt[n]),""===r||r?(o=parseFloat(a),r===!0||x.isNumeric(o)?o||0:a):a}}),e.getComputedStyle?(Rt=function(t){return e.getComputedStyle(t,null)},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s.getPropertyValue(n)||s[n]:t,u=e.style;return s&&(""!==l||x.contains(e.ownerDocument,e)||(l=x.style(e,n)),Yt.test(l)&&Ut.test(n)&&(i=u.width,o=u.minWidth,a=u.maxWidth,u.minWidth=u.maxWidth=u.width=l,l=s.width,u.width=i,u.minWidth=o,u.maxWidth=a)),l}):a.documentElement.currentStyle&&(Rt=function(e){return e.currentStyle},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s[n]:t,u=e.style;return null==l&&u&&u[n]&&(l=u[n]),Yt.test(l)&&!zt.test(n)&&(i=u.left,o=e.runtimeStyle,a=o&&o.left,a&&(o.left=e.currentStyle.left),u.left="fontSize"===n?"1em":l,l=u.pixelLeft+"px",u.left=i,a&&(o.left=a)),""===l?"auto":l});function on(e,t,n){var r=Vt.exec(t);return r?Math.max(0,r[1]-(n||0))+(r[2]||"px"):t}function an(e,t,n,r,i){var o=n===(r?"border":"content")?4:"width"===t?1:0,a=0;for(;4>o;o+=2)"margin"===n&&(a+=x.css(e,n+Zt[o],!0,i)),r?("content"===n&&(a-=x.css(e,"padding"+Zt[o],!0,i)),"margin"!==n&&(a-=x.css(e,"border"+Zt[o]+"Width",!0,i))):(a+=x.css(e,"padding"+Zt[o],!0,i),"padding"!==n&&(a+=x.css(e,"border"+Zt[o]+"Width",!0,i)));return a}function sn(e,t,n){var r=!0,i="width"===t?e.offsetWidth:e.offsetHeight,o=Rt(e),a=x.support.boxSizing&&"border-box"===x.css(e,"boxSizing",!1,o);if(0>=i||null==i){if(i=Wt(e,t,o),(0>i||null==i)&&(i=e.style[t]),Yt.test(i))return i;r=a&&(x.support.boxSizingReliable||i===e.style[t]),i=parseFloat(i)||0}return i+an(e,t,n||(a?"border":"content"),r,o)+"px"}function ln(e){var t=a,n=Gt[e];return n||(n=un(e,t),"none"!==n&&n||(Pt=(Pt||x("