diff --git a/README.md b/README.md index ef84a224f..c2e78e5a7 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Source code for all public APIs for com.google.appengine.api.* packages. com.google.appengine appengine-api-1.0-sdk - 2.0.12 + 2.0.13 javax.servlet @@ -131,7 +131,7 @@ Source code for remote APIs for App Engine. com.google.appengine appengine-remote-api - 2.0.12 + 2.0.13 ``` @@ -154,7 +154,7 @@ We moved `com.google.appengine.api.memcache.stdimpl` and its old dependency com.google.appengine appengine-api-legacy.jar/artifactId> - 2.0.12 + 2.0.13 ``` @@ -169,19 +169,19 @@ We moved `com.google.appengine.api.memcache.stdimpl` and its old dependency com.google.appengine appengine-testing - 2.0.12 + 2.0.13 test com.google.appengine appengine-api-stubs - 2.0.12 + 2.0.13 test com.google.appengine appengine-tools-sdk - 2.0.12 + 2.0.13 test ``` diff --git a/TRYLATESTBITSINPROD.md b/TRYLATESTBITSINPROD.md index 78a69558c..1e654c00d 100644 --- a/TRYLATESTBITSINPROD.md +++ b/TRYLATESTBITSINPROD.md @@ -43,12 +43,12 @@ top of your web application and change the entrypoint to boot with these jars in mvn clean install ``` -Let's assume the current built version is `2.0.13-SNAPSHOT`. +Let's assume the current built version is `2.0.14-SNAPSHOT`. Add the dependency for the GAE runtime jars in your application pom.xml file: ``` - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT ${appengine.runtime.location} ... diff --git a/api/pom.xml b/api/pom.xml index d3efe3afd..56eecf3d4 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT true diff --git a/api_dev/pom.xml b/api_dev/pom.xml index 58b149b55..89904513a 100644 --- a/api_dev/pom.xml +++ b/api_dev/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/api_legacy/pom.xml b/api_legacy/pom.xml index dbe9a8cac..68472be10 100644 --- a/api_legacy/pom.xml +++ b/api_legacy/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/appengine-api-1.0-sdk/pom.xml b/appengine-api-1.0-sdk/pom.xml index 1944339c5..3b3dd119c 100644 --- a/appengine-api-1.0-sdk/pom.xml +++ b/appengine-api-1.0-sdk/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar AppEngine :: appengine-api-1.0-sdk diff --git a/appengine-api-stubs/pom.xml b/appengine-api-stubs/pom.xml index 765c5d96d..2adbcbdbb 100644 --- a/appengine-api-stubs/pom.xml +++ b/appengine-api-stubs/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/appengine_jsr107/pom.xml b/appengine_jsr107/pom.xml index a26aad917..093942822 100644 --- a/appengine_jsr107/pom.xml +++ b/appengine_jsr107/pom.xml @@ -24,7 +24,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT diff --git a/appengine_resources/pom.xml b/appengine_resources/pom.xml index 6cb486b74..cce744a9f 100644 --- a/appengine_resources/pom.xml +++ b/appengine_resources/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar AppEngine :: appengine-resources diff --git a/appengine_testing/pom.xml b/appengine_testing/pom.xml index c197501bb..a03b8ded5 100644 --- a/appengine_testing/pom.xml +++ b/appengine_testing/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/appengine_testing_tests/pom.xml b/appengine_testing_tests/pom.xml index 3b8a74933..87faed4ff 100644 --- a/appengine_testing_tests/pom.xml +++ b/appengine_testing_tests/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/applications/pom.xml b/applications/pom.xml index c879ea9f8..b6669d851 100644 --- a/applications/pom.xml +++ b/applications/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT pom diff --git a/applications/proberapp/pom.xml b/applications/proberapp/pom.xml index 7c9493ff1..7a3033c21 100644 --- a/applications/proberapp/pom.xml +++ b/applications/proberapp/pom.xml @@ -27,7 +27,7 @@ com.google.appengine applications - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml index 91ce2d34c..0ec5cd837 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java index f75034292..e4dede29e 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java @@ -86,7 +86,7 @@ public class NoSerializeImmutableTest { public void serializableCollectionFieldsAreNotGuavaImmutable() throws Exception { File appengineApiJar = new File( - "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.13-SNAPSHOT.jar"); + "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.14-SNAPSHOT.jar"); assertThat(appengineApiJar.exists()).isTrue(); ClassLoader apiJarClassLoader = new URLClassLoader(new URL[] {appengineApiJar.toURI().toURL()}); Class messageLite = diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java index 948ef6f7e..b304a1d5e 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java @@ -54,7 +54,7 @@ public abstract class ApiExhaustiveUsageTestCase { /** The path to the sdk api jar. */ private static final String API_JAR_PATH = - "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.13-SNAPSHOT.jar"; + "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.14-SNAPSHOT.jar"; private boolean isExhaustiveUsageClass(String clsName) { return clsName.startsWith("com.google.appengine.apicompat.usage"); diff --git a/lib/pom.xml b/lib/pom.xml index cbe5e0085..3104e03e8 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT pom diff --git a/lib/tools_api/pom.xml b/lib/tools_api/pom.xml index 54fb746b0..fb4d62b59 100644 --- a/lib/tools_api/pom.xml +++ b/lib/tools_api/pom.xml @@ -23,7 +23,7 @@ com.google.appengine lib-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/lib/xml_validator/pom.xml b/lib/xml_validator/pom.xml index f5e4e8554..889272fa4 100644 --- a/lib/xml_validator/pom.xml +++ b/lib/xml_validator/pom.xml @@ -22,7 +22,7 @@ com.google.appengine lib-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar AppEngine :: libxmlvalidator diff --git a/lib/xml_validator_test/pom.xml b/lib/xml_validator_test/pom.xml index 2f5e095fe..ed78f7866 100644 --- a/lib/xml_validator_test/pom.xml +++ b/lib/xml_validator_test/pom.xml @@ -22,7 +22,7 @@ com.google.appengine lib-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar AppEngine :: libxmlvalidator_test diff --git a/local_runtime_shared/pom.xml b/local_runtime_shared/pom.xml index 56db6ea8d..8e83494fe 100644 --- a/local_runtime_shared/pom.xml +++ b/local_runtime_shared/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar AppEngine :: appengine-local-runtime-shared diff --git a/pom.xml b/pom.xml index 5eca51442..cb7301009 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 4.0.0 com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT pom AppEngine :: Parent project @@ -416,7 +416,7 @@ com.beust jcommander - 1.48 + 1.82 com.google.auto.service @@ -661,7 +661,7 @@ org.json json - 20220924 + 20230227 commons-codec diff --git a/protobuf/pom.xml b/protobuf/pom.xml index a5dab05d2..9fb6a57c2 100644 --- a/protobuf/pom.xml +++ b/protobuf/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/quickstartgenerator/pom.xml b/quickstartgenerator/pom.xml index a8c7383d7..22b7c8416 100644 --- a/quickstartgenerator/pom.xml +++ b/quickstartgenerator/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/remoteapi/pom.xml b/remoteapi/pom.xml index 6ce3f27af..e7300d7ff 100644 --- a/remoteapi/pom.xml +++ b/remoteapi/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar AppEngine :: appengine-remote-api diff --git a/runtime/deployment/pom.xml b/runtime/deployment/pom.xml index 4ad85753b..a689b59b3 100644 --- a/runtime/deployment/pom.xml +++ b/runtime/deployment/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT pom diff --git a/runtime/impl/pom.xml b/runtime/impl/pom.xml index f3e20c9de..01dac541e 100644 --- a/runtime/impl/pom.xml +++ b/runtime/impl/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java index 0bad7974d..df2d951ed 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/ApiProxyImpl.java @@ -161,7 +161,7 @@ public class ApiProxyImpl implements ApiProxy.Delegate getRequestThreads(EnvironmentImpl environment) { if (environment == null) { return Collections.emptyList(); } - return requestManager.getRequestThreads(environment.getAppVersion().getKey()); + return requestThreadManager.getRequestThreads(environment.getAppVersion().getKey()); } /** Creates an {@link Environment} instance that is suitable for use with this class. */ @@ -1132,6 +1132,7 @@ public String getVersionId() { * Get the trace id of the current request, which can be used to correlate log messages * belonging to that request. */ + @Override public Optional getTraceId() { return traceId; } @@ -1139,6 +1140,7 @@ public Optional getTraceId() { /** * Get the span id of the current request, which can be used to identify a span within a trace. */ + @Override public Optional getSpanId() { return spanId; } @@ -1244,7 +1246,7 @@ public TraceExceptionGenerator getTraceExceptionGenerator() { /** * A thread created by {@code ThreadManager.currentRequestThreadFactory(). */ - static class CurrentRequestThread extends Thread { + public static class CurrentRequestThread extends Thread { private final Runnable userRunnable; private final RequestState requestState; private final ApiProxy.Environment environment; @@ -1262,10 +1264,10 @@ static class CurrentRequestThread extends Thread { } /** - * Returns the original Runnable that was supplied to the thread factory, before any wrapping - * we may have done. + * Returns the original Runnable that was supplied to the thread factory, before any wrapping we + * may have done. */ - Runnable userRunnable() { + public Runnable userRunnable() { return userRunnable; } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/MutableUpResponse.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/MutableUpResponse.java index c509c1795..3e9a185c2 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/MutableUpResponse.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/MutableUpResponse.java @@ -80,11 +80,11 @@ public synchronized void setErrorMessage(String message) { builder.setErrorMessage(message); } - synchronized boolean hasSerializedTrace() { + public synchronized boolean hasSerializedTrace() { return builder.hasSerializedTrace(); } - synchronized ByteString getSerializedTrace() { + public synchronized ByteString getSerializedTrace() { return builder.getSerializedTrace(); } @@ -111,27 +111,27 @@ synchronized void addAppLog(AppLogsPb.AppLogLine line) { builder.addAppLog(line); } - synchronized void addAppLog(AppLogsPb.AppLogLine.Builder line) { + public synchronized void addAppLog(AppLogsPb.AppLogLine.Builder line) { builder.addAppLog(line); } - synchronized void setPendingCloudDebuggerActionBreakpointUpdates(boolean x) { + public synchronized void setPendingCloudDebuggerActionBreakpointUpdates(boolean x) { builder.getPendingCloudDebuggerActionBuilder().setBreakpointUpdates(x); } - synchronized void setPendingCloudDebuggerActionDebuggeeRegistration(boolean x) { + public synchronized void setPendingCloudDebuggerActionDebuggeeRegistration(boolean x) { builder.getPendingCloudDebuggerActionBuilder().setDebuggeeRegistration(x); } - synchronized boolean hasPendingCloudDebuggerAction() { + public synchronized boolean hasPendingCloudDebuggerAction() { return builder.hasPendingCloudDebuggerAction(); } - synchronized RuntimePb.PendingCloudDebuggerAction getPendingCloudDebuggerAction() { + public synchronized RuntimePb.PendingCloudDebuggerAction getPendingCloudDebuggerAction() { return builder.getPendingCloudDebuggerAction(); } - synchronized void setUserMcycles(long cycles) { + public synchronized void setUserMcycles(long cycles) { builder.setUserMcycles(cycles); } @@ -139,31 +139,31 @@ synchronized void addAllRuntimeLogLine(Collection lin builder.addAllRuntimeLogLine(lines); } - synchronized int getRuntimeLogLineCount() { + public synchronized int getRuntimeLogLineCount() { return builder.getRuntimeLogLineCount(); } - synchronized UPResponse.RuntimeLogLine getRuntimeLogLine(int i) { + public synchronized UPResponse.RuntimeLogLine getRuntimeLogLine(int i) { return builder.getRuntimeLogLine(i); } - synchronized boolean getTerminateClone() { + public synchronized boolean getTerminateClone() { return builder.getTerminateClone(); } - synchronized void setTerminateClone(boolean terminate) { + public synchronized void setTerminateClone(boolean terminate) { builder.setTerminateClone(terminate); } - synchronized boolean hasCloneIsInUncleanState() { + public synchronized boolean hasCloneIsInUncleanState() { return builder.hasCloneIsInUncleanState(); } - synchronized boolean getCloneIsInUncleanState() { + public synchronized boolean getCloneIsInUncleanState() { return builder.getCloneIsInUncleanState(); } - synchronized void setCloneIsInUncleanState(boolean unclean) { + public synchronized void setCloneIsInUncleanState(boolean unclean) { builder.setCloneIsInUncleanState(unclean); } diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/ParameterFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/ParameterFactory.java index d2ba9c860..94178bfb9 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/ParameterFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/ParameterFactory.java @@ -37,9 +37,8 @@ public class ParameterFactory implements IStringConverterFactory { Boolean.class, BooleanConverter.class); @Override - @SuppressWarnings("unchecked") - public Class> getConverter(Class type) { - return (Class>) CONVERTERS.get(type); + public Class> getConverter(Class type) { + return CONVERTERS.get(type); } /** diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java index 3aa730bbc..03a621c5a 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java @@ -69,19 +69,20 @@ import javax.annotation.Nullable; /** - * {@code RequestManager} is responsible for setting up and tearing - * down any state associated with each request. + * {@code RequestManager} is responsible for setting up and tearing down any state associated with + * each request. + * + *

At the moment, this includes: * - * At the moment, this includes: *

    - *
  • Injecting an {@code Environment} implementation for the - * request's thread into {@code ApiProxy}. - *
  • Scheduling any future actions that must occur while the - * request is executing (e.g. deadline exceptions), and cleaning up - * any scheduled actions that do not occur. + *
  • Injecting an {@code Environment} implementation for the request's thread into {@code + * ApiProxy}. + *
  • Scheduling any future actions that must occur while the request is executing (e.g. deadline + * exceptions), and cleaning up any scheduled actions that do not occur. *
* * It is expected that clients will use it like this: + * * 
  * RequestManager.RequestToken token =
  *     requestManager.startRequest(...);
@@ -91,9 +92,8 @@
  *   requestManager.finishRequest(token);
  * }
  *
- * */ -public class RequestManager { +public class RequestManager implements RequestThreadManager { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); /** @@ -929,7 +929,8 @@ public void shutdownRequests(RequestToken token) { response.setHttpResponseCodeAndResponse(200, "OK"); } - List getRequestThreads(AppVersionKey appVersionKey) { + @Override + public List getRequestThreads(AppVersionKey appVersionKey) { List threads = new ArrayList(); synchronized (requests) { for (RequestToken token : requests.values()) { diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestThreadManager.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestThreadManager.java new file mode 100644 index 000000000..2cff0080a --- /dev/null +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestThreadManager.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.apphosting.runtime; + +import com.google.apphosting.base.AppVersionKey; +import java.util.List; + +/** Thread manager interface for specifically getting request threads */ +public interface RequestThreadManager { + List getRequestThreads(AppVersionKey appVersionKey); +} diff --git a/runtime/lite/pom.xml b/runtime/lite/pom.xml index 8c726d806..22ded6c22 100644 --- a/runtime/lite/pom.xml +++ b/runtime/lite/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java index c223eb7c6..94981dd82 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/AppEngineRuntime.java @@ -25,7 +25,6 @@ import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.ApplicationEnvironment; -import com.google.apphosting.runtime.RequestManager; import com.google.apphosting.runtime.SessionsConfig; import com.google.apphosting.runtime.anyrpc.APIHostClientInterface; import com.google.apphosting.runtime.http.HttpApiHostClientFactory; @@ -178,7 +177,6 @@ public static Builder builderFromEnv(Map env) { apiProxyImpl = makeApiProxyImplBuilder(apiHostAddress, backgroundRequestDispatcher).build(); RequestManager requestManager = makeRequestManagerBuilder(apiProxyImpl).build(); - apiProxyImpl.setRequestManager(requestManager); AppInfoFactory appInfoFactory = new AppInfoFactory(System.getenv()); diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java index 70a6ed53f..08469b67e 100644 --- a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestHandler.java @@ -18,7 +18,6 @@ import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.MutableUpResponse; -import com.google.apphosting.runtime.RequestManager; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; import com.google.apphosting.runtime.jetty94.AppInfoFactory; import com.google.apphosting.runtime.jetty94.AppVersionHandlerFactory; diff --git a/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestManager.java b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestManager.java new file mode 100644 index 000000000..2f290b099 --- /dev/null +++ b/runtime/lite/src/main/java/com/google/appengine/runtime/lite/RequestManager.java @@ -0,0 +1,1213 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.runtime.lite; + +import static com.google.apphosting.base.protos.RuntimePb.UPRequest.Deadline.RPC_DEADLINE_PADDING_SECONDS_VALUE; +import static java.lang.Math.max; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.google.appengine.api.LifecycleManager; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.LogRecord.Level; +import com.google.apphosting.api.DeadlineExceededException; +import com.google.apphosting.base.AppVersionKey; +import com.google.apphosting.base.protos.AppLogsPb.AppLogLine; +import com.google.apphosting.base.protos.HttpPb; +import com.google.apphosting.base.protos.RuntimePb.UPRequest; +import com.google.apphosting.base.protos.RuntimePb.UPResponse; +import com.google.apphosting.runtime.ApiProxyImpl; +import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.CloudDebuggerAgentWrapper; +import com.google.apphosting.runtime.HardDeadlineExceededError; +import com.google.apphosting.runtime.MutableUpResponse; +import com.google.apphosting.runtime.RequestState; +import com.google.apphosting.runtime.RequestThreadManager; +import com.google.apphosting.runtime.RuntimeLogSink; +import com.google.apphosting.runtime.TraceWriter; +import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; +import com.google.apphosting.runtime.timer.CpuRatioTimer; +import com.google.apphosting.runtime.timer.JmxGcTimerSet; +import com.google.apphosting.runtime.timer.JmxHotspotTimerSet; +import com.google.apphosting.runtime.timer.TimerFactory; +import com.google.auto.value.AutoBuilder; +import com.google.common.base.Ascii; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.common.flogger.GoogleLogger; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + +/** + * {@code RequestManager} is responsible for setting up and tearing down any state associated with + * each request. + */ +public class RequestManager implements RequestThreadManager { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + /** The number of threads to use to execute scheduled {@code Future} actions. */ + private static final int SCHEDULER_THREADS = 1; + + // SimpleDateFormat is not threadsafe, so we'll just share the format string and let + // clients instantiate the format instances as-needed. At the moment the usage of the format + // objects shouldn't be too high volume, but if the construction of the format instance ever has + // a noticeable impact on performance (unlikely) we can switch to one format instance per thread + // using a ThreadLocal. + private static final String SIMPLE_DATE_FORMAT_STRING = "yyyy/MM/dd HH:mm:ss.SSS z"; + + /** The maximum number of stack frames to log for each thread when logging a deadlock. */ + private static final int MAXIMUM_DEADLOCK_STACK_LENGTH = 20; + + private static final ThreadMXBean THREAD_MX = ManagementFactory.getThreadMXBean(); + + private static final String INSTANCE_ID_ENV_ATTRIBUTE = "com.google.appengine.instance.id"; + + /** The amount of time to wait for pending async futures to cancel. */ + private static final Duration CANCEL_ASYNC_FUTURES_WAIT_TIME = Duration.ofMillis(150); + + /** + * The amount of time to wait for Thread.Interrupt to complete on all threads servicing a request. + */ + private static final Duration THREAD_INTERRUPT_WAIT_TIME = Duration.ofSeconds(1); + + private final long softDeadlineDelay; + private final long hardDeadlineDelay; + private final boolean disableDeadlineTimers; + private final ScheduledThreadPoolExecutor executor; + private final TimerFactory timerFactory; + private final Optional runtimeLogSink; + private final ApiProxyImpl apiProxyImpl; + private final boolean threadStopTerminatesClone; + private final Map requests; + private final boolean interruptFirstOnSoftDeadline; + private int maxOutstandingApiRpcs; + @Nullable private final CloudDebuggerAgentWrapper cloudDebuggerAgent; + private final AtomicBoolean enableCloudDebugger; + private final boolean waitForDaemonRequestThreads; + private final AtomicBoolean debugletStartNotified = new AtomicBoolean(false); + private final ImmutableMap environmentVariables; + + /** Make a partly-initialized builder for a RequestManager. */ + public static Builder builder() { + return new AutoBuilder_RequestManager_Builder().setEnvironment(System.getenv()); + } + + /** Builder for of a RequestManager instance. */ + @AutoBuilder + public abstract static class Builder { + Builder() {} + + public abstract Builder setSoftDeadlineDelay(long x); + + public abstract long softDeadlineDelay(); + + public abstract Builder setHardDeadlineDelay(long x); + + public abstract long hardDeadlineDelay(); + + public abstract Builder setDisableDeadlineTimers(boolean x); + + public abstract boolean disableDeadlineTimers(); + + public abstract Builder setRuntimeLogSink(Optional x); + + public abstract Builder setApiProxyImpl(ApiProxyImpl x); + + public abstract Builder setMaxOutstandingApiRpcs(int x); + + public abstract int maxOutstandingApiRpcs(); + + public abstract Builder setThreadStopTerminatesClone(boolean x); + + public abstract boolean threadStopTerminatesClone(); + + public abstract Builder setInterruptFirstOnSoftDeadline(boolean x); + + public abstract boolean interruptFirstOnSoftDeadline(); + + public abstract Builder setCloudDebuggerAgent(@Nullable CloudDebuggerAgentWrapper x); + + public abstract Builder setEnableCloudDebugger(boolean x); + + public abstract boolean enableCloudDebugger(); + + public abstract Builder setCyclesPerSecond(long x); + + public abstract long cyclesPerSecond(); + + public abstract Builder setWaitForDaemonRequestThreads(boolean x); + + public abstract boolean waitForDaemonRequestThreads(); + + public abstract Builder setEnvironment(Map x); + + public abstract RequestManager build(); + } + + RequestManager( + long softDeadlineDelay, + long hardDeadlineDelay, + boolean disableDeadlineTimers, + Optional runtimeLogSink, + ApiProxyImpl apiProxyImpl, + int maxOutstandingApiRpcs, + boolean threadStopTerminatesClone, + boolean interruptFirstOnSoftDeadline, + @Nullable CloudDebuggerAgentWrapper cloudDebuggerAgent, + boolean enableCloudDebugger, + long cyclesPerSecond, + boolean waitForDaemonRequestThreads, + ImmutableMap environment) { + + this.softDeadlineDelay = softDeadlineDelay; + this.hardDeadlineDelay = hardDeadlineDelay; + this.disableDeadlineTimers = disableDeadlineTimers; + this.executor = new ScheduledThreadPoolExecutor(SCHEDULER_THREADS); + this.timerFactory = + new TimerFactory(cyclesPerSecond, new JmxHotspotTimerSet(), new JmxGcTimerSet()); + this.runtimeLogSink = runtimeLogSink; + this.apiProxyImpl = apiProxyImpl; + this.maxOutstandingApiRpcs = maxOutstandingApiRpcs; + this.threadStopTerminatesClone = threadStopTerminatesClone; + this.interruptFirstOnSoftDeadline = interruptFirstOnSoftDeadline; + this.cloudDebuggerAgent = cloudDebuggerAgent; + this.enableCloudDebugger = new AtomicBoolean(enableCloudDebugger); + this.waitForDaemonRequestThreads = waitForDaemonRequestThreads; + this.requests = Collections.synchronizedMap(new HashMap()); + this.environmentVariables = environment; + } + + public void setMaxOutstandingApiRpcs(int maxOutstandingApiRpcs) { + this.maxOutstandingApiRpcs = maxOutstandingApiRpcs; + } + + /** + * Disables Cloud Debugger. + * + *

If called before the first request has been processed, the Cloud Debugger will not be even + * activated. + */ + public void disableCloudDebugger() { + enableCloudDebugger.set(false); + } + + /** + * Set up any state necessary to execute a new request using the specified parameters. The current + * thread should be the one that will execute the new request. + * + * @return a {@code RequestToken} that should be passed into {@code finishRequest} after the + * request completes. + */ + public RequestToken startRequest( + AppVersion appVersion, + AnyRpcServerContext rpc, + UPRequest upRequest, + MutableUpResponse upResponse, + ThreadGroup requestThreadGroup) { + long remainingTime = getAdjustedRpcDeadline(rpc, 60000); + long softDeadlineMillis = max(getAdjustedRpcDeadline(rpc, -1) - softDeadlineDelay, -1); + long millisUntilSoftDeadline = remainingTime - softDeadlineDelay; + Thread thread = Thread.currentThread(); + + // Hex-encode the request-id, formatted to 16 digits, in lower-case, + // with leading 0s, and no leading 0x to match the way stubby + // request ids are formatted in Google logs. + String requestId = String.format("%1$016x", rpc.getGlobalId()); + logger.atInfo().log("Beginning request %s remaining millis : %d", requestId, remainingTime); + + Runnable endAction; + if (isSnapshotRequest(upRequest)) { + logger.atInfo().log("Received snapshot request"); + endAction = new DisableApiHostAction(); + } else { + apiProxyImpl.enable(); + endAction = new NullAction(); + } + + TraceWriter traceWriter = TraceWriter.getTraceWriterForRequest(upRequest, upResponse); + if (traceWriter != null) { + URL requestURL = null; + try { + requestURL = new URL(upRequest.getRequest().getUrl()); + } catch (MalformedURLException e) { + logger.atWarning().withCause(e).log( + "Failed to extract path for trace due to malformed request URL: %s", + upRequest.getRequest().getUrl()); + } + if (requestURL != null) { + traceWriter.startRequestSpan(requestURL.getPath()); + } else { + traceWriter.startRequestSpan("Unparsable URL"); + } + } + + CpuRatioTimer timer = timerFactory.getCpuRatioTimer(thread); + + // This list is used to block the end of a request until all API + // calls have completed or timed out. + List> asyncFutures = Collections.synchronizedList(new ArrayList>()); + // This semaphore maintains the count of currently running API + // calls so we can block future calls if too many calls are + // outstanding. + Semaphore outstandingApiRpcSemaphore = new Semaphore(maxOutstandingApiRpcs); + + RequestState state = new RequestState(); + state.recordRequestThread(Thread.currentThread()); + + ApiProxy.Environment environment = + apiProxyImpl.createEnvironment( + appVersion, + upRequest, + upResponse, + traceWriter, + timer, + requestId, + asyncFutures, + outstandingApiRpcSemaphore, + requestThreadGroup, + state, + millisUntilSoftDeadline); + + // If the instance id was not set (e.g. for some Titanium runtimes), set the instance id + // retrieved from the environment variable. + String instanceId = environmentVariables.get("GAE_INSTANCE"); + if (!Strings.isNullOrEmpty(instanceId)) { + environment.getAttributes().putIfAbsent(INSTANCE_ID_ENV_ATTRIBUTE, instanceId); + } + + // Create a RequestToken where we will store any state we will + // need to restore in finishRequest(). + RequestToken token = + new RequestToken( + thread, + upResponse, + requestId, + upRequest.getSecurityTicket(), + timer, + asyncFutures, + appVersion, + softDeadlineMillis, + rpc, + rpc.getStartTimeMillis(), + traceWriter, + state, + endAction); + + requests.put(upRequest.getSecurityTicket(), token); + + // Tell the ApiProxy about our current request environment so that + // it can make callbacks and pass along information about the + // logged-in user. + ApiProxy.setEnvironmentForCurrentThread(environment); + + // Let the appserver know that we're up and running. + setPendingStartCloudDebugger(upResponse); + + // Start counting CPU cycles used by this thread. + timer.start(); + + if (!disableDeadlineTimers) { + // The timing conventions here are a bit wonky, but this is what + // the Python runtime does. + logger.atInfo().log( + "Scheduling soft deadline in %d ms for %s", millisUntilSoftDeadline, requestId); + token.addScheduledFuture( + schedule(new DeadlineRunnable(this, token, false), millisUntilSoftDeadline)); + } + + return token; + } + + /** + * Tear down any state associated with the specified request, and restore the current thread's + * state as it was before {@code startRequest} was called. + * + * @throws IllegalStateException if called from the wrong thread. + */ + public void finishRequest(RequestToken requestToken) { + verifyRequestAndThread(requestToken); + + // Don't let user code create any more threads. This is + // especially important for ThreadPoolExecutors, which will try to + // backfill the threads that we're about to interrupt without user + // intervention. + requestToken.getState().setAllowNewRequestThreadCreation(false); + + // Interrupt any other request threads. + for (Thread thread : getActiveThreads(requestToken)) { + logger.atWarning().log("Interrupting %s", thread); + thread.interrupt(); + } + + // Send any pending breakpoint updates from Cloud Debugger. + if (enableCloudDebugger.get() && cloudDebuggerAgent.hasBreakpointUpdates()) { + setPendingCloudDebuggerBreakpointUpdates(requestToken.getUpResponse()); + } + + // Now wait for any async API calls and all request threads to complete. + waitForUserCodeToComplete(requestToken); + + // There is no more user code left, stop the timers and tear down the state. + requests.remove(requestToken.getSecurityTicket()); + requestToken.setFinished(); + + // Stop the timer first so the user does get charged for our clean-up. + CpuRatioTimer timer = requestToken.getRequestTimer(); + timer.stop(); + + // Cancel any scheduled future actions associated with this + // request. + // + // N.B.: Copy the list to avoid a + // ConcurrentModificationException due to a race condition where + // the soft deadline runnable runs and adds the hard deadline + // runnable while we are waiting for it to finish. We don't + // actually care about this race because we set + // RequestToken.finished above and both runnables check that + // first. + for (Future future : new ArrayList>(requestToken.getScheduledFutures())) { + // Unit tests will fail if a future fails to execute correctly, but + // we won't get a good error message if it was due to some exception. + // Log a future failure due to exception here. + if (future.isDone()) { + try { + future.get(); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.atSevere().withCause(e).log("Future failed execution: %s", future); + } + } else if (future.cancel(false)) { + logger.atFine().log("Removed scheduled future: %s", future); + } else { + logger.atFine().log("Unable to remove scheduled future: %s", future); + } + } + + // Store the CPU usage for this request in the UPResponse. + logger.atInfo().log("Stopped timer for request %s %s", requestToken.getRequestId(), timer); + requestToken.getUpResponse().setUserMcycles(timer.getCycleCount() / 1000000L); + + if (requestToken.getTraceWriter() != null) { + requestToken.getTraceWriter().endRequestSpan(); + requestToken.getTraceWriter().flushTrace(); + } + + requestToken.runEndAction(); + + // Remove our environment information to remove any potential + // for leakage. + ApiProxy.clearEnvironmentForCurrentThread(); + + runtimeLogSink.ifPresent(x -> x.flushLogs(requestToken.getUpResponse())); + } + + private static boolean isSnapshotRequest(UPRequest request) { + try { + URI uri = new URI(request.getRequest().getUrl()); + if (!uri.getPath().equals("/_ah/snapshot")) { + return false; + } + } catch (URISyntaxException e) { + return false; + } + for (HttpPb.ParsedHttpHeader header : request.getRequest().getHeadersList()) { + if (Ascii.equalsIgnoreCase("X-AppEngine-Snapshot", header.getKey())) { + return true; + } + } + return false; + } + + private class DisableApiHostAction implements Runnable { + @Override + public void run() { + apiProxyImpl.disable(); + } + } + + // In Java 8, the method Thread.stop(Throwable), which has been deprecated for about 15 years, + // has finally been disabled. It now throws UnsupportedOperationException. However, Thread.stop() + // still works, and calls the JNI Method Thread.stop0(Object) with a Throwable argument. + // So at least for the time being we can still achieve the effect of Thread.stop(Throwable) by + // calling the JNI method. That means we don't get the permission checks and so on that come + // with Thread.stop, but the code that's calling it is privileged anyway. + private static class ThreadStop0Holder { + private static final Method threadStop0; + + static { + try { + threadStop0 = Thread.class.getDeclaredMethod("stop0", Object.class); + threadStop0.setAccessible(true); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + } + + private static class NullAction implements Runnable { + @Override + public void run() {} + } + + public void sendDeadline(String securityTicket, boolean isUncatchable) { + logger.atInfo().log("Looking up token for security ticket %s", securityTicket); + sendDeadline(requests.get(securityTicket), isUncatchable); + } + + // Although Thread.stop(Throwable) is deprecated due to being + // "inherently unsafe", it does exactly what we want. Locks will + // still be released (unlike Thread.destroy), so the only real + // risk is that users are not expecting a particular piece of code + // to throw an exception, and therefore when an exception is + // thrown it leaves their objects in a bad state. Since objects + // should not be shared across requests, this should not be a very + // big problem. + public void sendDeadline(RequestToken token, boolean isUncatchable) { + if (token == null) { + logger.atInfo().log("No token, can't send deadline"); + return; + } + checkForDeadlocks(token); + + final Thread targetThread = token.getRequestThread(); + logger.atInfo().log( + "Sending deadline: %s, %s, %b", targetThread, token.getRequestId(), isUncatchable); + + if (interruptFirstOnSoftDeadline && !isUncatchable) { + // Disable thread creation and cancel all pending futures, then interrupt all threads, + // all while giving the application some time to return a response after each step. + token.getState().setAllowNewRequestThreadCreation(false); + cancelPendingAsyncFutures(token.getAsyncFutures()); + waitForResponseDuringSoftDeadline(CANCEL_ASYNC_FUTURES_WAIT_TIME); + if (!token.isFinished()) { + logger.atInfo().log("Interrupting all request threads."); + for (Thread thread : getActiveThreads(token)) { + thread.interrupt(); + } + // Runtime will kill the clone if all threads servicing the request + // are not interrupted by the end of this wait. This is set to 2s as + // a reasonable amount of time to interrupt the maximum number of threads (50). + waitForResponseDuringSoftDeadline(THREAD_INTERRUPT_WAIT_TIME); + } + } + + if (isUncatchable) { + token.getState().setHardDeadlinePassed(true); + } else { + token.getState().setSoftDeadlinePassed(true); + } + + // Only resort to Thread.stop on a soft deadline if all the prior nudging + // failed to elicit a response. On hard deadlines, there is no nudging. + if (!token.isFinished()) { + // SimpleDateFormat isn't threadsafe so just instantiate as-needed + final DateFormat dateFormat = new SimpleDateFormat(SIMPLE_DATE_FORMAT_STRING, Locale.US); + // Give the user as much information as we can. + final Throwable throwable = + createDeadlineThrowable( + "This request (" + + token.getRequestId() + + ") " + + "started at " + + dateFormat.format(token.getStartTimeMillis()) + + " and was still executing at " + + dateFormat.format(System.currentTimeMillis()) + + ".", + isUncatchable); + // There is a weird race condition here. We're throwing an + // exception during the execution of an arbitrary method, but + // that exception will contain the stack trace of what the + // thread was doing a very small amount of time *before* the + // exception was thrown (i.e. potentially in a different method). + // + // TODO: Add a try-catch block to every instrumented + // method, which catches this throwable (or an internal version + // of it) and checks to see if the stack trace has the proper + // class and method at the top. If so, rethrow it (or a public + // version of it). If not, create a new exception with the + // correct class and method, but with an unknown line number. + // + // N.B.: Also, we're now using this stack trace to + // determine when to terminate the clone. The above issue may + // cause us to terminate either too many or two few clones. Too + // many is merely wasteful, and too few is no worse than it was + // without this change. + boolean terminateClone = false; + StackTraceElement[] stackTrace = targetThread.getStackTrace(); + if (threadStopTerminatesClone || isUncatchable || inClassInitialization(stackTrace)) { + // If we bypassed catch blocks or interrupted class + // initialization, don't reuse this clone. + terminateClone = true; + } + + throwable.setStackTrace(stackTrace); + + // Check again, since calling Thread.stop is so harmful. + if (!token.isFinished()) { + // Only set this if we're absolutely determined to call Thread.stop. + token.getUpResponse().setTerminateClone(terminateClone); + if (terminateClone) { + token.getUpResponse().setCloneIsInUncleanState(true); + } + logger.atInfo().log("Stopping request thread."); + // Throw the exception in targetThread. + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + try { + ThreadStop0Holder.threadStop0.invoke(targetThread, throwable); + } catch (Exception e) { + logger.atWarning().withCause(e).log("Failed to stop thread"); + } + return null; + }); + } + } + } + + private void setPendingStartCloudDebugger(MutableUpResponse upResponse) { + if (!enableCloudDebugger.get()) { + return; + } + + // First time ever we need to set "DebugletStarted" flag. This will trigger + // debuggee initialization sequence on AppServer. + if (debugletStartNotified.compareAndSet(false, true)) { + upResponse.setPendingCloudDebuggerActionDebuggeeRegistration(true); + } + } + + private void setPendingCloudDebuggerBreakpointUpdates(MutableUpResponse upResponse) { + if (!enableCloudDebugger.get()) { + return; + } + + upResponse.setPendingCloudDebuggerActionBreakpointUpdates(true); + } + + private String threadDump(Collection threads, String prefix) { + StringBuilder message = new StringBuilder(prefix); + for (Thread thread : threads) { + message.append(thread).append(" in state ").append(thread.getState()).append("\n"); + StackTraceElement[] stack = thread.getStackTrace(); + if (stack.length == 0) { + message.append("... empty stack\n"); + } else { + for (StackTraceElement element : thread.getStackTrace()) { + message.append("... ").append(element).append("\n"); + } + } + message.append("\n"); + } + return message.toString(); + } + + private void waitForUserCodeToComplete(RequestToken requestToken) { + RequestState state = requestToken.getState(); + if (Thread.interrupted()) { + logger.atInfo().log("Interrupt bit set in waitForUserCodeToComplete, resetting."); + // interrupted() already reset the bit. + } + + try { + if (state.hasHardDeadlinePassed()) { + logger.atInfo().log("Hard deadline has already passed; skipping wait for async futures."); + } else { + // Wait for async API calls to complete. Don't bother doing + // this if the hard deadline has already passed, we're not going to + // reuse this JVM anyway. + waitForPendingAsyncFutures(requestToken.getAsyncFutures()); + } + + // Now wait for any request-scoped threads to complete. + Set threads; + while (!(threads = getActiveThreads(requestToken)).isEmpty()) { + if (state.hasHardDeadlinePassed()) { + requestToken.getUpResponse().setError(UPResponse.ERROR.THREADS_STILL_RUNNING_VALUE); + requestToken.getUpResponse().clearHttpResponse(); + String messageString = threadDump(threads, "Thread(s) still running after request:\n"); + logger.atWarning().log("%s", messageString); + requestToken.addAppLogMessage(ApiProxy.LogRecord.Level.fatal, messageString); + return; + } else { + try { + // Interrupt the threads one more time before joining. + // This helps with ThreadPoolExecutors, where the first + // interrupt may cancel the current runnable but another + // interrupt is needed to kill the (now-idle) worker + // thread. + for (Thread thread : threads) { + thread.interrupt(); + } + if (Boolean.getBoolean("com.google.appengine.force.thread.pool.shutdown")) { + attemptThreadPoolShutdown(threads); + } + for (Thread thread : threads) { + logger.atInfo().log("Waiting for completion of thread: %s", thread); + // Initially wait up to 10 seconds. If the interrupted thread takes longer than that + // to stop then it's probably not going to. We will wait for it anyway, in case it + // does stop, but we'll also log what the threads we are waiting for are doing. + thread.join(10_000); + if (thread.isAlive()) { + // We're probably going to block forever. + String message = threadDump(threads, "Threads still running after 10 seconds:\n"); + logger.atWarning().log("%s", message); + requestToken.addAppLogMessage(ApiProxy.LogRecord.Level.warn, message); + thread.join(); + } + } + logger.atInfo().log("All request threads have completed."); + } catch (DeadlineExceededException ex) { + // expected, try again. + } catch (HardDeadlineExceededError ex) { + // expected, loop around and we'll do something else this time. + } + } + } + } catch (Throwable ex) { + if (ex instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.atWarning().withCause(ex).log( + "Exception thrown while waiting for background work to complete:"); + } + } + + /** + * Scans the given threads to see if any of them looks like a ThreadPoolExecutor thread that was + * created using {@link com.google.appengine.api.ThreadManager#currentRequestThreadFactory()}, and + * if so attempts to shut down the owning ThreadPoolExecutor. + */ + private void attemptThreadPoolShutdown(Collection threads) { + for (Thread t : threads) { + if (t instanceof ApiProxyImpl.CurrentRequestThread) { + // This thread was made by ThreadManager.currentRequestThreadFactory. Check what Runnable + // it was given. + Runnable runnable = ((ApiProxyImpl.CurrentRequestThread) t).userRunnable(); + if (runnable + .getClass() + .getName() + .equals("java.util.concurrent.ThreadPoolExecutor$Worker")) { + // This is the class that ThreadPoolExecutor threads use as their Runnable. + // This check depends on implementation details of the JDK, and could break in the future. + // In that case we have tests that should fail. + // Assuming it is indeed a ThreadPoolExecutor.Worker, and given that that is an inner + // class, we should be able to access the enclosing ThreadPoolExecutor instance by + // accessing the synthetic this$0 field. That is again dependent on the JDK + // implementation. + try { + Field outerField = runnable.getClass().getDeclaredField("this$0"); + outerField.setAccessible(true); + Object outer = outerField.get(runnable); + if (outer instanceof ThreadPoolExecutor) { + ThreadPoolExecutor executor = (ThreadPoolExecutor) outer; + executor.shutdown(); + // We might already have seen this executor via another thread in the loop, but + // there's no harm in telling it more than once to shut down. + } + } catch (ReflectiveOperationException e) { + logger.atInfo().withCause(e).log("ThreadPoolExecutor reflection failed"); + } + } + } + } + } + + private void waitForPendingAsyncFutures(Collection> asyncFutures) + throws InterruptedException { + int size = asyncFutures.size(); + if (size > 0) { + logger.atWarning().log("Waiting for %d pending async futures.", size); + List> snapshot; + synchronized (asyncFutures) { + snapshot = new ArrayList<>(asyncFutures); + } + for (Future future : snapshot) { + // Unlike scheduled futures, we do *not* want to cancel these + // futures if they aren't done yet. They represent asynchronous + // actions that the user began which we still want to succeed. + // We simply need to block until they do. + try { + // Don't bother specifying a deadline -- + // DeadlineExceededException's will break us out of here if + // necessary. + future.get(); + } catch (ExecutionException ex) { + logger.atInfo().withCause(ex.getCause()).log("Async future failed:"); + } + } + // Note that it's possible additional futures have been added to asyncFutures while + // we were waiting, and they will not get waited for. It's also possible additional + // futures could be added after this method returns. There's nothing to prevent that. + // For now we are keeping this loophole in order to avoid the risk of incompatibility + // with existing apps. + logger.atWarning().log("Done waiting for pending async futures."); + } + } + + private void cancelPendingAsyncFutures(Collection> asyncFutures) { + int size = asyncFutures.size(); + if (size > 0) { + logger.atInfo().log("Canceling %d pending async futures.", size); + List> snapshot; + synchronized (asyncFutures) { + snapshot = new ArrayList<>(asyncFutures); + } + for (Future future : snapshot) { + future.cancel(true); + } + logger.atInfo().log("Done canceling pending async futures."); + } + } + + private void waitForResponseDuringSoftDeadline(Duration responseWaitTimeMs) { + try { + Thread.sleep(responseWaitTimeMs.toMillis()); + } catch (InterruptedException e) { + logger.atInfo().withCause(e).log( + "Interrupted while waiting for response during soft deadline"); + } + } + + /** + * Returns all the threads belonging to the current request except the current thread. For + * compatibility, on Java 7 this returns all threads in the same thread group as the original + * request thread. On later Java versions this returns the original request thread plus all + * threads that were created with {@code ThreadManager.currentRequestThreadFactory()} and that + * have not yet terminated. + */ + private Set getActiveThreads(RequestToken token) { + Set threads; + if (waitForDaemonRequestThreads) { + // Join all request threads created using the current request ThreadFactory, including + // daemon ones. + threads = token.getState().requestThreads(); + } else { + // Join all live non-daemon request threads created using the current request ThreadFactory. + Set nonDaemonThreads = new LinkedHashSet<>(); + for (Thread thread : token.getState().requestThreads()) { + if (thread.isDaemon()) { + logger.atInfo().log("Ignoring daemon thread: %s", thread); + } else if (!thread.isAlive()) { + logger.atInfo().log("Ignoring dead thread: %s", thread); + } else { + nonDaemonThreads.add(thread); + } + } + threads = nonDaemonThreads; + } + Set activeThreads = new LinkedHashSet<>(threads); + activeThreads.remove(Thread.currentThread()); + return activeThreads; + } + + /** + * Check that the current thread matches the one that called startRequest. + * + * @throws IllegalStateException If called from the wrong thread. + */ + private void verifyRequestAndThread(RequestToken requestToken) { + if (requestToken.getRequestThread() != Thread.currentThread()) { + throw new IllegalStateException( + "Called from " + + Thread.currentThread() + + ", should be " + + requestToken.getRequestThread()); + } + } + + /** Arrange for the specified {@code Runnable} to be executed in {@code time} milliseconds. */ + private Future schedule(Runnable runnable, long time) { + logger.atFine().log("Scheduling %s to run in %d ms.", runnable, time); + return executor.schedule(runnable, time, MILLISECONDS); + } + + /** + * Adjusts the deadline for this RPC by the padding constant along with the elapsed time. Will + * return the defaultValue if the rpc is not valid. + */ + private long getAdjustedRpcDeadline(AnyRpcServerContext rpc, long defaultValue) { + if (rpc.getTimeRemaining().compareTo(Duration.ofNanos(Long.MAX_VALUE)) >= 0 + || rpc.getStartTimeMillis() == 0) { + logger.atWarning().log( + "Did not receive enough RPC information to calculate adjusted deadline: %s", rpc); + return defaultValue; + } + + long elapsedMillis = System.currentTimeMillis() - rpc.getStartTimeMillis(); + + if (rpc.getTimeRemaining().compareTo(Duration.ofSeconds(RPC_DEADLINE_PADDING_SECONDS_VALUE)) + < 0) { + logger.atWarning().log("RPC deadline is less than padding. Not adjusting deadline"); + return rpc.getTimeRemaining().minusMillis(elapsedMillis).toMillis(); + } else { + return rpc.getTimeRemaining() + .minusSeconds(RPC_DEADLINE_PADDING_SECONDS_VALUE) + .minusMillis(elapsedMillis) + .toMillis(); + } + } + + /** Notify requests that the server is shutting down. */ + public void shutdownRequests(RequestToken token) { + checkForDeadlocks(token); + logger.atInfo().log("Calling shutdown hooks for %s", token.getAppVersionKey()); + // TODO what if there's other app/versions in this VM? + MutableUpResponse response = token.getUpResponse(); + + // Set the context classloader to the UserClassLoader while invoking the + // shutdown hooks. + ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(token.getAppVersion().getClassLoader()); + try { + LifecycleManager.getInstance().beginShutdown(token.getDeadline()); + } finally { + Thread.currentThread().setContextClassLoader(oldClassLoader); + } + + logMemoryStats(); + + logAllStackTraces(); + + response.setError(UPResponse.ERROR.OK_VALUE); + response.setHttpResponseCodeAndResponse(200, "OK"); + } + + @Override + public List getRequestThreads(AppVersionKey appVersionKey) { + List threads = new ArrayList<>(); + synchronized (requests) { + for (RequestToken token : requests.values()) { + if (appVersionKey.equals(token.getAppVersionKey())) { + threads.add(token.getRequestThread()); + } + } + } + return threads; + } + + /** + * Consults {@link ThreadMXBean#findDeadlockedThreads()} to see if any deadlocks are currently + * present. If so, it will immediately respond to the runtime and simulate a LOG(FATAL) containing + * the stack trace of the offending threads. + */ + private void checkForDeadlocks(final RequestToken token) { + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + long[] deadlockedThreadsIds = THREAD_MX.findDeadlockedThreads(); + if (deadlockedThreadsIds != null) { + StringBuilder builder = new StringBuilder(); + builder.append( + "Detected a deadlock across ") + .append(deadlockedThreadsIds.length) + .append(" threads:"); + for (ThreadInfo info : + THREAD_MX.getThreadInfo(deadlockedThreadsIds, MAXIMUM_DEADLOCK_STACK_LENGTH)) { + builder.append(info); + builder.append("\n"); + } + String message = builder.toString(); + token.addAppLogMessage(Level.fatal, message); + token.logAndKillRuntime(message); + } + return null; + }); + } + + private void logMemoryStats() { + Runtime runtime = Runtime.getRuntime(); + logger.atInfo().log( + "maxMemory=%d totalMemory=%d freeMemory=%d", + runtime.maxMemory(), runtime.totalMemory(), runtime.freeMemory()); + } + + private void logAllStackTraces() { + AccessController.doPrivileged( + (PrivilegedAction) + () -> { + long[] allthreadIds = THREAD_MX.getAllThreadIds(); + StringBuilder builder = new StringBuilder(); + builder + .append("Dumping thread info for all ") + .append(allthreadIds.length) + .append(" runtime threads:"); + for (ThreadInfo info : + THREAD_MX.getThreadInfo(allthreadIds, MAXIMUM_DEADLOCK_STACK_LENGTH)) { + builder.append(info); + builder.append("\n"); + } + String message = builder.toString(); + logger.atInfo().log("%s", message); + return null; + }); + } + + private Throwable createDeadlineThrowable(String message, boolean isUncatchable) { + if (isUncatchable) { + return new HardDeadlineExceededError(message); + } else { + return new DeadlineExceededException(message); + } + } + + private boolean inClassInitialization(StackTraceElement[] stackTrace) { + for (StackTraceElement element : stackTrace) { + if (element.getMethodName().equals("")) { + return true; + } + } + return false; + } + + /** + * {@code RequestToken} acts as a Memento object that passes state between a call to {@code + * startRequest} and {@code finishRequest}. It should be treated as opaque by clients. + */ + public static class RequestToken { + /** + * The thread of the request. This is used to verify that {@code finishRequest} was called from + * the right thread. + */ + private final Thread requestThread; + + private final MutableUpResponse upResponse; + + /** + * A collection of {@code Future} objects that have been scheduled on behalf of this request. + * These futures will each be cancelled when the request completes. + */ + private final List> scheduledFutures; + + private final Collection> asyncFutures; + + private final String requestId; + + private final String securityTicket; + + /** + * A {@code Timer} that runs during the course of the request and measures both wallclock and + * CPU time. + */ + private final CpuRatioTimer requestTimer; + + @Nullable private final TraceWriter traceWriter; + + private volatile boolean finished; + + private final AppVersion appVersion; + + private final long deadline; + + private final AnyRpcServerContext rpc; + private final long startTimeMillis; + + private final RequestState state; + private final Runnable endAction; + + RequestToken( + Thread requestThread, + MutableUpResponse upResponse, + String requestId, + String securityTicket, + CpuRatioTimer requestTimer, + Collection> asyncFutures, + AppVersion appVersion, + long deadline, + AnyRpcServerContext rpc, + long startTimeMillis, + @Nullable TraceWriter traceWriter, + RequestState state, + Runnable endAction) { + this.requestThread = requestThread; + this.upResponse = upResponse; + this.requestId = requestId; + this.securityTicket = securityTicket; + this.requestTimer = requestTimer; + this.asyncFutures = asyncFutures; + this.scheduledFutures = new ArrayList<>(); + this.finished = false; + this.appVersion = appVersion; + this.deadline = deadline; + this.rpc = rpc; + this.startTimeMillis = startTimeMillis; + this.traceWriter = traceWriter; + this.state = state; + this.endAction = endAction; + } + + public RequestState getState() { + return state; + } + + Thread getRequestThread() { + return requestThread; + } + + MutableUpResponse getUpResponse() { + return upResponse; + } + + CpuRatioTimer getRequestTimer() { + return requestTimer; + } + + public String getRequestId() { + return requestId; + } + + public String getSecurityTicket() { + return securityTicket; + } + + public AppVersion getAppVersion() { + return appVersion; + } + + public AppVersionKey getAppVersionKey() { + return appVersion.getKey(); + } + + public long getDeadline() { + return deadline; + } + + public long getStartTimeMillis() { + return startTimeMillis; + } + + Collection> getScheduledFutures() { + return scheduledFutures; + } + + void addScheduledFuture(Future future) { + scheduledFutures.add(future); + } + + Collection> getAsyncFutures() { + return asyncFutures; + } + + @Nullable + TraceWriter getTraceWriter() { + return traceWriter; + } + + boolean isFinished() { + return finished; + } + + void setFinished() { + finished = true; + } + + public void addAppLogMessage(ApiProxy.LogRecord.Level level, String message) { + upResponse.addAppLog( + AppLogLine.newBuilder() + .setLevel(level.ordinal()) + .setTimestampUsec(System.currentTimeMillis() * 1000) + .setMessage(message)); + } + + void logAndKillRuntime(String errorMessage) { + logger.atSevere().log("LOG(FATAL): %s", errorMessage); + upResponse.clearHttpResponse(); + upResponse.setError(UPResponse.ERROR.LOG_FATAL_DEATH_VALUE); + upResponse.setErrorMessage(errorMessage); + rpc.finishWithResponse(upResponse.build()); + } + + void runEndAction() { + endAction.run(); + } + } + + /** + * {@code DeadlineRunnable} causes the specified {@code Throwable} to be thrown within the + * specified thread. The stack trace of the Throwable is ignored, and is replaced with the stack + * trace of the thread at the time the exception is thrown. + */ + public class DeadlineRunnable implements Runnable { + private final RequestManager requestManager; + private final RequestToken token; + private final boolean isUncatchable; + + public DeadlineRunnable( + RequestManager requestManager, RequestToken token, boolean isUncatchable) { + this.requestManager = requestManager; + this.token = token; + this.isUncatchable = isUncatchable; + } + + @Override + public void run() { + requestManager.sendDeadline(token, isUncatchable); + + if (!token.isFinished()) { + if (!isUncatchable) { + token.addScheduledFuture( + schedule( + new DeadlineRunnable(requestManager, token, true), + softDeadlineDelay - hardDeadlineDelay)); + } + + logger.atInfo().log("Finished execution of %s", this); + } + } + + @Override + public String toString() { + return "DeadlineRunnable(" + + token.getRequestThread() + + ", " + + token.getRequestId() + + ", " + + isUncatchable + + ")"; + } + } +} diff --git a/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java index 5d5fdd873..f9d2e35e8 100644 --- a/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java +++ b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/AppEngineRuntimeTest.java @@ -35,7 +35,6 @@ import com.google.apphosting.runtime.ApiProxyImpl; import com.google.apphosting.runtime.AppVersion; import com.google.apphosting.runtime.ApplicationEnvironment; -import com.google.apphosting.runtime.RequestManager; import com.google.apphosting.runtime.SessionsConfig; import com.google.apphosting.runtime.http.FakeHttpApiHost; import com.google.apphosting.runtime.jetty94.AppInfoFactory; diff --git a/runtime/lite/src/test/java/com/google/appengine/runtime/lite/RequestManagerTest.java b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/RequestManagerTest.java new file mode 100644 index 000000000..1664c9fa5 --- /dev/null +++ b/runtime/lite/src/test/java/com/google/appengine/runtime/lite/RequestManagerTest.java @@ -0,0 +1,564 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.runtime.lite; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.DeadlineExceededException; +import com.google.apphosting.base.AppVersionKey; +import com.google.apphosting.base.protos.AppinfoPb.AppInfo; +import com.google.apphosting.base.protos.RuntimePb.UPRequest; +import com.google.apphosting.base.protos.RuntimePb.UPResponse; +import com.google.apphosting.base.protos.SpanKindOuterClass; +import com.google.apphosting.base.protos.TraceEvents.SpanEventProto; +import com.google.apphosting.base.protos.TraceEvents.SpanEventsProto; +import com.google.apphosting.base.protos.TraceEvents.StartSpanProto; +import com.google.apphosting.base.protos.TraceEvents.TraceEventsProto; +import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.ApiProxyImpl; +import com.google.apphosting.runtime.AppVersion; +import com.google.apphosting.runtime.ApplicationEnvironment; +import com.google.apphosting.runtime.CloudDebuggerAgentWrapper; +import com.google.apphosting.runtime.MutableUpResponse; +import com.google.apphosting.runtime.RuntimeLogSink; +import com.google.apphosting.runtime.SessionsConfig; +import com.google.apphosting.runtime.anyrpc.APIHostClientInterface; +import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; +import com.google.apphosting.runtime.test.MockAnyRpcServerContext; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.flogger.GoogleLogger; +import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.ByteString; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.InvalidProtocolBufferException; +import java.io.File; +import java.nio.file.Files; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Optional; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +/** Unit tests for the RequestManager. */ +@RunWith(JUnit4.class) +public class RequestManagerTest { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + private static final double RPC_DEADLINE = 3.0; + private static final long SLEEP_TIME = 5000; + private static final long SOFT_DEADLINE_DELAY = 750; + private static final long HARD_DEADLINE_DELAY = 250; + private static final long MAX_RUNTIME_LOG_PER_REQUEST = 3000L * 1024L; + private static final String APP_ID = "app123"; + private static final String ENGINE_ID = "engine"; + private static final String VERSION_ID = "v456"; + private static final long CYCLES_PER_SECOND = 2333414000L; + private static final String INSTANCE_ID_ENV_ATTRIBUTE = "com.google.appengine.instance.id"; + private static final String INSTANCE_ID = "abc123"; + + private AppVersion appVersion; + private UPRequest upRequest; + private MutableUpResponse upResponse; + private RuntimeLogSink logSink; + @Mock private CloudDebuggerAgentWrapper cloudDebuggerAgent; + @Mock private APIHostClientInterface mockApiHost; + + // Ensure that Truth is loaded. Otherwise we can get weird errors if the exceptions we are + // flinging about with Thread.stop0 end up hitting a thread that is running the Truth static + // initializer. Likewise for Mockito. + @BeforeClass + public static void initClasses() { + assertThat(true).isTrue(); + Mockito.mock(CloudDebuggerAgentWrapper.class).hasBreakpointUpdates(); + } + + private static class DeadlineThread extends Thread { + private final RequestManager requestManager; + private final RequestManager.RequestToken token; + private final boolean isUncatchable; + private final CountDownLatch started = new CountDownLatch(1); + + private DeadlineThread( + RequestManager requestManager, RequestManager.RequestToken token, boolean isUncatchable) { + this.requestManager = requestManager; + this.token = token; + this.isUncatchable = isUncatchable; + } + + static void startAndWait( + RequestManager requestManager, RequestManager.RequestToken token, boolean isUncatchable) { + DeadlineThread thread = new DeadlineThread(requestManager, token, isUncatchable); + thread.start(); + try { + // Wait for the thread's run() method to start. + thread.started.await(); + // Further small sleep to allow the run() method to progress before we return. + Thread.sleep(1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public void run() { + started.countDown(); + requestManager.sendDeadline(token, isUncatchable); + } + } + + @Before + public void setUp() throws Exception { + + MockitoAnnotations.openMocks(this); + + upRequest = + UPRequest.newBuilder() + .setAppId(APP_ID) + .setModuleId(ENGINE_ID) + .setModuleVersionId(VERSION_ID) + .buildPartial(); + + upResponse = new MutableUpResponse(); + logSink = new RuntimeLogSink(MAX_RUNTIME_LOG_PER_REQUEST); + + File rootDirectory = Files.createTempDirectory("appengine").toFile(); + ApplicationEnvironment appEnv = + new ApplicationEnvironment( + APP_ID, + VERSION_ID, + ImmutableMap.of(), + ImmutableMap.of(), + rootDirectory, + ApplicationEnvironment.RuntimeConfiguration.DEFAULT_FOR_TEST); + appVersion = + AppVersion.builder() + .setAppVersionKey(AppVersionKey.of(APP_ID, VERSION_ID)) + .setAppInfo(AppInfo.getDefaultInstance()) + .setRootDirectory(rootDirectory) + .setEnvironment(appEnv) + .setSessionsConfig(new SessionsConfig(false, false, null)) + .setPublicRoot("") + .build(); + } + + private RequestManager.Builder requestManagerBuilder() { + return RequestManager.builder() + .setSoftDeadlineDelay(SOFT_DEADLINE_DELAY) + .setHardDeadlineDelay(HARD_DEADLINE_DELAY) + .setRuntimeLogSink(Optional.of(logSink)) + .setApiProxyImpl(ApiProxyImpl.builder().setApiHost(mockApiHost).build()) + .setMaxOutstandingApiRpcs(10) + .setCloudDebuggerAgent(cloudDebuggerAgent) + .setCyclesPerSecond(CYCLES_PER_SECOND) + .setWaitForDaemonRequestThreads(true) + .setDisableDeadlineTimers(false) + .setThreadStopTerminatesClone(true) + .setInterruptFirstOnSoftDeadline(false) + .setEnableCloudDebugger(true); + } + + private RequestManager createRequestManager() { + return requestManagerBuilder().build(); + } + + @Test + public void testApiEnvironment() { + RequestManager requestManager = createRequestManager(); + assertThat(ApiProxy.getCurrentEnvironment()).isEqualTo(null); + + MockAnyRpcServerContext rpc = createRpc(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + try { + assertThat(ApiProxy.getCurrentEnvironment().getAppId()).isEqualTo(APP_ID); + assertThat(ApiProxy.getCurrentEnvironment().getModuleId()).isEqualTo(ENGINE_ID); + assertThat(ApiProxy.getCurrentEnvironment().getVersionId()).isEqualTo(VERSION_ID); + assertThat(ApiProxy.getCurrentEnvironment().getAttributes().get(INSTANCE_ID_ENV_ATTRIBUTE)) + .isNull(); + } finally { + requestManager.finishRequest(token); + } + assertThat(ApiProxy.getCurrentEnvironment()).isEqualTo(null); + } + + @Test + public void testApiEnvironmentWithInstanceIdFromEnvironmentVariables() { + RequestManager requestManager = + requestManagerBuilder() + .setEnvironment(ImmutableMap.of("GAE_INSTANCE", INSTANCE_ID)) + .build(); + assertThat(ApiProxy.getCurrentEnvironment()).isEqualTo(null); + + MockAnyRpcServerContext rpc = createRpc(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + try { + assertThat(ApiProxy.getCurrentEnvironment().getAttributes().get(INSTANCE_ID_ENV_ATTRIBUTE)) + .isEqualTo(INSTANCE_ID); + } finally { + requestManager.finishRequest(token); + } + assertThat(ApiProxy.getCurrentEnvironment()).isEqualTo(null); + } + + // Outcome of the request thread in the next test (testSoftExceptionWithInterruption). + enum TestOutcome { + NONE("Unexpected outcome"), + OK("OK"), + THREAD_NOT_INTERRUPTED("Thread slept past the allotted deadline"), + ASYNC_FUTURE_NOT_CANCELLED("Async future was not cancelled"), + DEADLINE_THROWN("Thread was not interrupted, instead got a DeadlineExceededException"); + + TestOutcome(String message) { + this.message = message; + } + + String getMessage() { + return message; + } + + private final String message; + } + + @Test + public void testSoftExceptionWithInterruption() throws Exception { + RequestManager requestManager = + requestManagerBuilder() + .setInterruptFirstOnSoftDeadline(true) + .setEnableCloudDebugger(false) + .build(); + MockAnyRpcServerContext rpc = createRpc(); + ThreadGroup threadGroup = new ThreadGroup("test-interruption"); + AtomicReference outcome = new AtomicReference<>(TestOutcome.NONE); + // We cannot simply use the thread from the test environment, because interruption + // only applies to the request thread group. Note also that startRequest and finishRequest + // must be called by the same thread. + Thread t = + new Thread( + threadGroup, + () -> doTestSoftExceptionWithInterruption(requestManager, rpc, threadGroup, outcome)); + t.start(); + t.join(); + if (outcome.get() != TestOutcome.OK) { + fail(outcome.get().getMessage()); + } + assertThat(upResponse.getTerminateClone()).isFalse(); + } + + private void doTestSoftExceptionWithInterruption( + RequestManager requestManager, + AnyRpcServerContext rpc, + ThreadGroup threadGroup, + AtomicReference outcome) { + RequestManager.RequestToken token = + requestManager.startRequest(appVersion, rpc, upRequest, upResponse, threadGroup); + Future asyncFuture = SettableFuture.create(); + token.getAsyncFutures().add(asyncFuture); + try { + try { + asyncFuture.get(SLEEP_TIME, MILLISECONDS); + } catch (CancellationException e) { + // Expected. + } catch (InterruptedException | ExecutionException | TimeoutException e) { + // Unexpected or downright impossible cases. Do nothing and let the outcome + // be set by the test for cancellation. + } + if (!asyncFuture.isCancelled()) { + outcome.set(TestOutcome.ASYNC_FUTURE_NOT_CANCELLED); + } else { + // Now test the second step, thread interruption. + try { + Thread.sleep(SLEEP_TIME); + outcome.set(TestOutcome.THREAD_NOT_INTERRUPTED); + } catch (InterruptedException ex) { + // All went as expected. + outcome.set(TestOutcome.OK); + } + } + } catch (DeadlineExceededException ex) { + outcome.set(TestOutcome.DEADLINE_THROWN); + } finally { + requestManager.finishRequest(token); + } + } + + @Test + public void testTraceDisabled() { + RequestManager requestManager = createRequestManager(); + MockAnyRpcServerContext rpc = createRpc(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + requestManager.finishRequest(token); + assertThat(upResponse.hasSerializedTrace()).isFalse(); + } + + @Test + public void testTraceEnabled() throws InvalidProtocolBufferException { + RequestManager requestManager = createRequestManager(); + // Enable trace. + TraceContextProto context = + TraceContextProto.newBuilder() + .setTraceId(ByteString.copyFromUtf8("trace id")) + .setSpanId(1L) + .setTraceMask(1) + .build(); + UPRequest.Builder upRequestBuilder = upRequest.toBuilder().setTraceContext(context); + upRequestBuilder.getRequestBuilder().setUrl("http://foo.com/request?a=1"); + upRequest = upRequestBuilder.buildPartial(); + + MockAnyRpcServerContext rpc = createRpc(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + // Construct failed response. + upResponse.setError(UPResponse.ERROR.LOG_FATAL_DEATH_VALUE); + upResponse.setErrorMessage("Error message"); + requestManager.finishRequest(token); + + TraceEventsProto traceEvents = + TraceEventsProto.parseFrom( + upResponse.getSerializedTrace(), ExtensionRegistry.getGeneratedRegistry()); + + assertThat(traceEvents.getSpanEventsCount()).isEqualTo(1); + + // Verify request span. + SpanEventsProto spanEvents = traceEvents.getSpanEvents(0); + assertThat(spanEvents.getSpanId().hasId()).isTrue(); + assertThat(spanEvents.getEventCount()).isEqualTo(2); + SpanEventProto startSpanEvent = spanEvents.getEvent(0); + StartSpanProto startSpan = startSpanEvent.getStartSpan(); + assertThat(startSpan.getKind()).isEqualTo(SpanKindOuterClass.SpanKind.RPC_SERVER); + assertThat(startSpan.getName()).isEqualTo("/request"); + assertThat(startSpan.getParentSpanId().getId()).isEqualTo(1L); + SpanEventProto endSpanEvent = spanEvents.getEvent(1); + assertThat(endSpanEvent.getTimestamp()).isAtLeast(startSpanEvent.getTimestamp()); + } + + @Test + public void testTraceEnabledBadURL() throws InvalidProtocolBufferException { + RequestManager requestManager = createRequestManager(); + // Enable trace. + TraceContextProto context = + TraceContextProto.newBuilder() + .setTraceId(ByteString.copyFromUtf8("trace id")) + .setSpanId(1L) + .setTraceMask(1) + .build(); + UPRequest.Builder upRequestBuilder = upRequest.toBuilder().setTraceContext(context); + upRequestBuilder.getRequestBuilder().setUrl("foo.com/request?a=1"); + upRequest = upRequestBuilder.buildPartial(); + + MockAnyRpcServerContext rpc = createRpc(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + requestManager.finishRequest(token); + + TraceEventsProto traceEvents = + TraceEventsProto.parseFrom( + upResponse.getSerializedTrace(), ExtensionRegistry.getGeneratedRegistry()); + StartSpanProto startSpan = traceEvents.getSpanEvents(0).getEvent(0).getStartSpan(); + assertThat(startSpan.getName()).isEqualTo("Unparsable URL"); + } + + @Test + public void testRuntimeLogging() { + RequestManager requestManager = createRequestManager(); + MockAnyRpcServerContext rpc = createRpc(); + String prefix = "com.google.apphosting.runtime.RequestManagerTest testRuntimeLogging: "; + Deque messages = + new ArrayDeque<>( + ImmutableList.of( + "Before startRequest.\n", + "INFO During request.\n", + "WARNING During request.\n", + "ERROR During request.\n", + "After finishRequest.\n")); + Deque levels = + new ArrayDeque<>( + ImmutableList.of( + 0, // INFO + 0, // INFO + 1, // WARNING + 2, // ERROR + 0 // INFO + )); + + logger.atInfo().log("Before startRequest."); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + logger.atInfo().log("INFO During request."); + logger.atWarning().log("WARNING During request."); + logger.atSevere().log("ERROR During request."); + requestManager.finishRequest(token); + logger.atInfo().log("After finishRequest."); + + for (int i = 0; i < upResponse.getRuntimeLogLineCount(); i++) { + String message = upResponse.getRuntimeLogLine(i).getMessage(); + if (message.startsWith(prefix)) { + assertThat(message.substring(prefix.length())).isEqualTo(messages.removeFirst()); + assertThat(upResponse.getRuntimeLogLine(i).getSeverity()) + .isEqualTo(((int) levels.removeFirst())); + } + } + } + + @Test + public void testCloudDebugger() { + RequestManager requestManager = createRequestManager(); + ApiProxy.Delegate delegate = mock(ApiProxy.Delegate.class); + ApiProxy.setDelegate(delegate); + + try { + // Prepare return values of 3 calls to hasBreakpointUpdates() + // as we simulate 3 sequential requests. + when(cloudDebuggerAgent.hasBreakpointUpdates()) + .thenReturn(false) + .thenReturn(false) + .thenReturn(true); + + // 1st request + MockAnyRpcServerContext rpc = createRpc(); + MutableUpResponse upResponse1 = new MutableUpResponse(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse1, new ThreadGroup("test")); + requestManager.finishRequest(token); + + // Validation of PendingCloudDebuggerAction in UPResponse + // - debuggee initialization is set + // - no breakpoint updates from hasBreakpointUpdates() + Mockito.verifyNoMoreInteractions(delegate); + assertThat(upResponse1.hasPendingCloudDebuggerAction()).isTrue(); + assertThat(upResponse1.getPendingCloudDebuggerAction().getDebuggeeRegistration()).isTrue(); + assertThat(upResponse1.getPendingCloudDebuggerAction().getBreakpointUpdates()).isFalse(); + + // 2nd request + rpc = createRpc(); + MutableUpResponse upResponse2 = new MutableUpResponse(); + token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse2, new ThreadGroup("test")); + requestManager.finishRequest(token); + + // Validation of PendingCloudDebuggerAction in UPResponse + // - no PendingCloudDebuggerAction since + // - no debuggee initialization (this is not 1st request) + // - no breakpoint updates from hasBreakpointUpdates() + Mockito.verifyNoMoreInteractions(delegate); + assertThat(upResponse2.hasPendingCloudDebuggerAction()).isFalse(); + + // 3rd request + rpc = createRpc(); + MutableUpResponse upResponse3 = new MutableUpResponse(); + token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse3, new ThreadGroup("test")); + requestManager.finishRequest(token); + + // Validation of PendingCloudDebuggerAction in UPResponse + // - no debuggee initialization (this is not 1st request) + // - pending breakpoint updates from hasBreakpointUpdates() + Mockito.verifyNoMoreInteractions(delegate); + assertThat(upResponse3.hasPendingCloudDebuggerAction()).isTrue(); + assertThat(upResponse3.getPendingCloudDebuggerAction().getDebuggeeRegistration()).isFalse(); + assertThat(upResponse3.getPendingCloudDebuggerAction().getBreakpointUpdates()).isTrue(); + + } finally { + ApiProxy.setDelegate(null); + } + } + + @Test + public void testCloudDebuggerDisabled() { + RequestManager requestManager = requestManagerBuilder().setEnableCloudDebugger(false).build(); + + ApiProxy.Delegate delegate = mock(ApiProxy.Delegate.class); + ApiProxy.setDelegate(delegate); + + try { + MockAnyRpcServerContext rpc = createRpc(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + requestManager.finishRequest(token); + + // Validate there's no PendingCloudDebuggerAction in UPResponse + Mockito.verifyNoMoreInteractions(cloudDebuggerAgent); + Mockito.verifyNoMoreInteractions(delegate); + assertThat(upResponse.hasPendingCloudDebuggerAction()).isFalse(); + } finally { + ApiProxy.setDelegate(null); + } + } + + @Test + public void testCloudDebuggerApplicationDisabled() { + // Cloud debugger is initially enabled. + RequestManager.Builder builder = requestManagerBuilder(); + assertThat(builder.enableCloudDebugger()).isTrue(); + RequestManager requestManager = builder.build(); + + // Now programmatically disable Cloud Debugger. + requestManager.disableCloudDebugger(); + + // Verify that the Cloud Debugger is indeed disabled on RequestManager. + ApiProxy.Delegate delegate = mock(ApiProxy.Delegate.class); + ApiProxy.setDelegate(delegate); + + try { + MockAnyRpcServerContext rpc = createRpc(); + RequestManager.RequestToken token = + requestManager.startRequest( + appVersion, rpc, upRequest, upResponse, new ThreadGroup("test")); + requestManager.finishRequest(token); + + Mockito.verifyNoMoreInteractions(cloudDebuggerAgent); + Mockito.verifyNoMoreInteractions(delegate); + assertThat(upResponse.hasPendingCloudDebuggerAction()).isFalse(); + } finally { + ApiProxy.setDelegate(null); + } + } + + private MockAnyRpcServerContext createRpc() { + return new MockAnyRpcServerContext(Duration.ofNanos(Math.round(RPC_DEADLINE * 1e9))); + } +} diff --git a/runtime/local/pom.xml b/runtime/local/pom.xml index cbcee1acb..fd20b830e 100644 --- a/runtime/local/pom.xml +++ b/runtime/local/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/runtime/main/pom.xml b/runtime/main/pom.xml index 899b4a3ca..6807aa927 100644 --- a/runtime/main/pom.xml +++ b/runtime/main/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/runtime/pom.xml b/runtime/pom.xml index 25449ccee..517f8c38b 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT AppEngine :: runtime projects pom diff --git a/runtime/test/pom.xml b/runtime/test/pom.xml index 95578feaa..3d53dcc0f 100644 --- a/runtime/test/pom.xml +++ b/runtime/test/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/runtime/testapps/pom.xml b/runtime/testapps/pom.xml index 5d3d24965..4531782c1 100644 --- a/runtime/testapps/pom.xml +++ b/runtime/testapps/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/runtime/util/pom.xml b/runtime/util/pom.xml index 739e468a9..b50ae3a15 100644 --- a/runtime/util/pom.xml +++ b/runtime/util/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/runtime_shared/pom.xml b/runtime_shared/pom.xml index b4b93d404..fc8152225 100644 --- a/runtime_shared/pom.xml +++ b/runtime_shared/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/sdk_assembly/pom.xml b/sdk_assembly/pom.xml index f43810223..a963fd110 100644 --- a/sdk_assembly/pom.xml +++ b/sdk_assembly/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT 4.0.0 appengine-java-sdk diff --git a/sessiondata/pom.xml b/sessiondata/pom.xml index c12c321df..48848c21f 100644 --- a/sessiondata/pom.xml +++ b/sessiondata/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/shared_sdk/pom.xml b/shared_sdk/pom.xml index 97d76e2d2..c4cd02686 100644 --- a/shared_sdk/pom.xml +++ b/shared_sdk/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT jar diff --git a/third_party/geronimo_javamail/pom.xml b/third_party/geronimo_javamail/pom.xml index 74a3cd1b2..9e66932ec 100644 --- a/third_party/geronimo_javamail/pom.xml +++ b/third_party/geronimo_javamail/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT ../../pom.xml diff --git a/utils/pom.xml b/utils/pom.xml index 3ef22f033..ee5a4a706 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.13-SNAPSHOT + 2.0.14-SNAPSHOT true