From 54be42b8e25af0906436959edef470ebe03c8773 Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Mon, 17 Oct 2022 08:32:10 -0700 Subject: [PATCH 01/50] Refactor UPRequest and UPResponse translation for reusability PiperOrigin-RevId: 481647516 Change-Id: I7716a44554eeab42f5ed54f56a65db183468a91d --- .../runtime/jetty94/JettyHttpProxy.java | 388 +---------------- .../runtime/jetty94/UPRequestTranslator.java | 412 ++++++++++++++++++ 2 files changed, 419 insertions(+), 381 deletions(-) create mode 100644 runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/UPRequestTranslator.java diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java index 0d5492147..e66b8dd73 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java @@ -19,45 +19,31 @@ import com.google.apphosting.base.protos.AppLogsPb; import com.google.apphosting.base.protos.AppinfoPb; import com.google.apphosting.base.protos.EmptyMessage; -import com.google.apphosting.base.protos.HttpPb; -import com.google.apphosting.base.protos.HttpPb.HttpRequest; -import com.google.apphosting.base.protos.HttpPb.ParsedHttpHeader; import com.google.apphosting.base.protos.RuntimePb; import com.google.apphosting.base.protos.RuntimePb.UPRequest; import com.google.apphosting.base.protos.RuntimePb.UPResponse; -import com.google.apphosting.base.protos.TracePb.TraceContextProto; import com.google.apphosting.runtime.ServletEngineAdapter; -import com.google.apphosting.runtime.TraceContextHelper; import com.google.apphosting.runtime.anyrpc.AnyRpcServerContext; import com.google.apphosting.runtime.anyrpc.EvaluationRuntimeServerInterface; import com.google.common.base.Ascii; -import com.google.common.base.Strings; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableSet; import com.google.common.flogger.GoogleLogger; -import com.google.common.html.HtmlEscapers; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.SettableFuture; -import com.google.protobuf.ByteString; import com.google.protobuf.MessageLite; -import com.google.protobuf.TextFormat; import java.io.IOException; import java.time.Duration; -import java.util.Collections; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpCompliance; -import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.Response; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.AbstractHandler; @@ -111,7 +97,7 @@ public static Server newServer( new JettyServerConnectorWithReusePort(server, runtimeOptions.jettyReusePort()); c.setHost(runtimeOptions.jettyHttpAddress().getHost()); c.setPort(runtimeOptions.jettyHttpAddress().getPort()); - server.setConnectors(new Connector[]{c}); + server.setConnectors(new Connector[] {c}); HttpConnectionFactory factory = c.getConnectionFactory(HttpConnectionFactory.class); factory.setHttpCompliance( @@ -192,79 +178,13 @@ public long getStartTimeMillis() { // The class has to be public, as it is a Servlet that needs to be loaded by the Jetty server. public static class ForwardingHandler extends AbstractHandler { - private static final String DEFAULT_SECRET_KEY = "secretkey"; - - - /** - * The HTTP headers that are handled specially by this proxy are defined in lowercae - * because HTTP headers are case insensitive and we look then up in a set or switch after - * converting to lower-case. - */ - private static final String X_FORWARDED_PROTO = "x-forwarded-proto"; - private static final String X_APPENGINE_API_TICKET = "x-appengine-api-ticket"; - private static final String X_APPENGINE_HTTPS = "x-appengine-https"; - private static final String X_APPENGINE_USER_IP = "x-appengine-user-ip"; - private static final String X_APPENGINE_USER_EMAIL = "x-appengine-user-email"; - private static final String X_APPENGINE_AUTH_DOMAIN = "x-appengine-auth-domain"; - private static final String X_APPENGINE_USER_ID = "x-appengine-user-id"; - private static final String X_APPENGINE_USER_NICKNAME = "x-appengine-user-nickname"; - private static final String X_APPENGINE_USER_ORGANIZATION = "x-appengine-user-organization"; - private static final String X_APPENGINE_USER_IS_ADMIN = "x-appengine-user-is-admin"; - private static final String X_APPENGINE_TRUSTED_IP_REQUEST = "x-appengine-trusted-ip-request"; - private static final String X_APPENGINE_LOAS_PEER_USERNAME = "x-appengine-loas-peer-username"; - private static final String X_APPENGINE_GAIA_ID = "x-appengine-gaia-id"; - private static final String X_APPENGINE_GAIA_AUTHUSER = "x-appengine-gaia-authuser"; - private static final String X_APPENGINE_GAIA_SESSION = "x-appengine-gaia-session"; - private static final String X_APPENGINE_APPSERVER_DATACENTER = - "x-appengine-appserver-datacenter"; - private static final String X_APPENGINE_APPSERVER_TASK_BNS = "x-appengine-appserver-task-bns"; - private static final String X_APPENGINE_DEFAULT_VERSION_HOSTNAME = - "x-appengine-default-version-hostname"; - private static final String X_APPENGINE_REQUEST_LOG_ID = "x-appengine-request-log-id"; - private static final String X_APPENGINE_QUEUENAME = "x-appengine-queuename"; private static final String X_APPENGINE_TIMEOUT_MS = "x-appengine-timeout-ms"; - private static final String X_GOOGLE_INTERNAL_SKIPADMINCHECK = - "x-google-internal-skipadmincheck"; - private static final String X_GOOGLE_INTERNAL_SKIPADMINCHECK_UC = - "X-Google-Internal-SkipAdminCheck"; - private static final String X_GOOGLE_INTERNAL_PROFILER = "x-google-internal-profiler"; - private static final String X_CLOUD_TRACE_CONTEXT = "x-cloud-trace-context"; - - private static final String IS_ADMIN_HEADER_VALUE = "1"; - private static final String IS_TRUSTED = "1"; - - // The impersonated IP address of warmup requests (and also background) - // () - private static final String WARMUP_IP = "0.1.0.3"; - - private static final ImmutableSet PRIVATE_APPENGINE_HEADERS = - ImmutableSet.of( - X_APPENGINE_API_TICKET, - X_APPENGINE_HTTPS, - X_APPENGINE_USER_IP, - X_APPENGINE_USER_EMAIL, - X_APPENGINE_AUTH_DOMAIN, - X_APPENGINE_USER_ID, - X_APPENGINE_USER_NICKNAME, - X_APPENGINE_USER_ORGANIZATION, - X_APPENGINE_USER_IS_ADMIN, - X_APPENGINE_TRUSTED_IP_REQUEST, - X_APPENGINE_LOAS_PEER_USERNAME, - X_APPENGINE_GAIA_ID, - X_APPENGINE_GAIA_AUTHUSER, - X_APPENGINE_GAIA_SESSION, - X_APPENGINE_APPSERVER_DATACENTER, - X_APPENGINE_APPSERVER_TASK_BNS, - X_APPENGINE_DEFAULT_VERSION_HOSTNAME, - X_APPENGINE_REQUEST_LOG_ID, - X_APPENGINE_TIMEOUT_MS, - X_GOOGLE_INTERNAL_PROFILER); private final String applicationRoot; private final String fixedApplicationPath; private final AppInfoFactory appInfoFactory; private final EvaluationRuntimeServerInterface evaluationRuntimeServerInterface; - private final boolean passThroughPrivateHeaders; + private final UPRequestTranslator upRequestTranslator; public ForwardingHandler(ServletEngineAdapter.Config runtimeOptions, Map env) throws ExecutionException, InterruptedException, IOException { @@ -272,7 +192,8 @@ public ForwardingHandler(ServletEngineAdapter.Config runtimeOptions, MapServer Error"); - outstr.print("" + HtmlEscapers.htmlEscaper().escape(errMsg) + ""); - } catch (IOException iox) { - throw new RuntimeException(iox); - } - } - } - - private static HttpPb.ParsedHttpHeader.Builder createRuntimeHeader(String key, String value) { - return HttpPb.ParsedHttpHeader.newBuilder().setKey(key).setValue(value); } private static Level toJavaLevel(long level) { diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/UPRequestTranslator.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/UPRequestTranslator.java new file mode 100644 index 000000000..ecdbff80f --- /dev/null +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/UPRequestTranslator.java @@ -0,0 +1,412 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty94; + +import com.google.apphosting.base.protos.AppinfoPb; +import com.google.apphosting.base.protos.HttpPb; +import com.google.apphosting.base.protos.HttpPb.HttpRequest; +import com.google.apphosting.base.protos.HttpPb.ParsedHttpHeader; +import com.google.apphosting.base.protos.RuntimePb; +import com.google.apphosting.base.protos.RuntimePb.UPRequest; +import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.apphosting.runtime.TraceContextHelper; +import com.google.common.base.Ascii; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.GoogleLogger; +import com.google.common.html.HtmlEscapers; +import com.google.protobuf.ByteString; +import com.google.protobuf.TextFormat; +import java.io.IOException; +import java.util.Collections; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; + +/** Translates HttpServletRequest to the UPRequest proto, and vice versa for the response. */ +public class UPRequestTranslator { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + private static final String DEFAULT_SECRET_KEY = "secretkey"; + + /** + * The HTTP headers that are handled specially by this proxy are defined in lowercae because HTTP + * headers are case insensitive and we look then up in a set or switch after converting to + * lower-case. + */ + private static final String X_FORWARDED_PROTO = "x-forwarded-proto"; + + private static final String X_APPENGINE_API_TICKET = "x-appengine-api-ticket"; + private static final String X_APPENGINE_HTTPS = "x-appengine-https"; + private static final String X_APPENGINE_USER_IP = "x-appengine-user-ip"; + private static final String X_APPENGINE_USER_EMAIL = "x-appengine-user-email"; + private static final String X_APPENGINE_AUTH_DOMAIN = "x-appengine-auth-domain"; + private static final String X_APPENGINE_USER_ID = "x-appengine-user-id"; + private static final String X_APPENGINE_USER_NICKNAME = "x-appengine-user-nickname"; + private static final String X_APPENGINE_USER_ORGANIZATION = "x-appengine-user-organization"; + private static final String X_APPENGINE_USER_IS_ADMIN = "x-appengine-user-is-admin"; + private static final String X_APPENGINE_TRUSTED_IP_REQUEST = "x-appengine-trusted-ip-request"; + private static final String X_APPENGINE_LOAS_PEER_USERNAME = "x-appengine-loas-peer-username"; + private static final String X_APPENGINE_GAIA_ID = "x-appengine-gaia-id"; + private static final String X_APPENGINE_GAIA_AUTHUSER = "x-appengine-gaia-authuser"; + private static final String X_APPENGINE_GAIA_SESSION = "x-appengine-gaia-session"; + private static final String X_APPENGINE_APPSERVER_DATACENTER = "x-appengine-appserver-datacenter"; + private static final String X_APPENGINE_APPSERVER_TASK_BNS = "x-appengine-appserver-task-bns"; + private static final String X_APPENGINE_DEFAULT_VERSION_HOSTNAME = + "x-appengine-default-version-hostname"; + private static final String X_APPENGINE_REQUEST_LOG_ID = "x-appengine-request-log-id"; + private static final String X_APPENGINE_QUEUENAME = "x-appengine-queuename"; + private static final String X_APPENGINE_TIMEOUT_MS = "x-appengine-timeout-ms"; + private static final String X_GOOGLE_INTERNAL_SKIPADMINCHECK = "x-google-internal-skipadmincheck"; + private static final String X_GOOGLE_INTERNAL_SKIPADMINCHECK_UC = + "X-Google-Internal-SkipAdminCheck"; + private static final String X_GOOGLE_INTERNAL_PROFILER = "x-google-internal-profiler"; + private static final String X_CLOUD_TRACE_CONTEXT = "x-cloud-trace-context"; + + private static final String IS_ADMIN_HEADER_VALUE = "1"; + private static final String IS_TRUSTED = "1"; + + // The impersonated IP address of warmup requests (and also background) + // () + private static final String WARMUP_IP = "0.1.0.3"; + + private static final ImmutableSet PRIVATE_APPENGINE_HEADERS = + ImmutableSet.of( + X_APPENGINE_API_TICKET, + X_APPENGINE_HTTPS, + X_APPENGINE_USER_IP, + X_APPENGINE_USER_EMAIL, + X_APPENGINE_AUTH_DOMAIN, + X_APPENGINE_USER_ID, + X_APPENGINE_USER_NICKNAME, + X_APPENGINE_USER_ORGANIZATION, + X_APPENGINE_USER_IS_ADMIN, + X_APPENGINE_TRUSTED_IP_REQUEST, + X_APPENGINE_LOAS_PEER_USERNAME, + X_APPENGINE_GAIA_ID, + X_APPENGINE_GAIA_AUTHUSER, + X_APPENGINE_GAIA_SESSION, + X_APPENGINE_APPSERVER_DATACENTER, + X_APPENGINE_APPSERVER_TASK_BNS, + X_APPENGINE_DEFAULT_VERSION_HOSTNAME, + X_APPENGINE_REQUEST_LOG_ID, + X_APPENGINE_TIMEOUT_MS, + X_GOOGLE_INTERNAL_PROFILER); + + private final AppInfoFactory appInfoFactory; + private final boolean passThroughPrivateHeaders; + + public UPRequestTranslator(AppInfoFactory appInfoFactory, boolean passThroughPrivateHeaders) { + this.appInfoFactory = appInfoFactory; + this.passThroughPrivateHeaders = passThroughPrivateHeaders; + } + + /** + * Translate from a response proto to a javax.servlet response. + * + * @param response the Jetty response object to fill + * @param rpcResp the proto info available to extract info from + */ + public final void translateResponse(Response response, RuntimePb.UPResponse rpcResp) { + HttpPb.HttpResponse rpcHttpResp = rpcResp.getHttpResponse(); + + if (rpcResp.getError() != RuntimePb.UPResponse.ERROR.OK.getNumber()) { + populateErrorResponse(response, "Request failed: " + rpcResp.getErrorMessage()); + return; + } + response.setStatus(rpcHttpResp.getResponsecode()); + for (HttpPb.ParsedHttpHeader header : rpcHttpResp.getOutputHeadersList()) { + response.addHeader(header.getKey(), header.getValue()); + } + + try { + response.getHttpOutput().sendContent(rpcHttpResp.getResponse().asReadOnlyByteBuffer()); + } catch (IOException ex) { + throw new IllegalStateException(ex); + } + } + + /** + * Makes a UPRequest from an HttpServletRequest + * + * @param realRequest the http request object + * @return equivalent UPRequest object + */ + @SuppressWarnings("JdkObsolete") + public final RuntimePb.UPRequest translateRequest(HttpServletRequest realRequest) { + UPRequest.Builder upReqBuilder = + UPRequest.newBuilder() + .setAppId(appInfoFactory.getGaeApplication()) + .setVersionId(appInfoFactory.getGaeVersion()) + .setModuleId(appInfoFactory.getGaeService()) + .setModuleVersionId(appInfoFactory.getGaeServiceVersion()); + + // TODO(b/78515194) Need to find a mapping for all these upReqBuilder fields: + /* + setRequestLogId(); + setEventIdHash(); + setSecurityLevel()); + */ + + upReqBuilder.setSecurityTicket(DEFAULT_SECRET_KEY); + upReqBuilder.setNickname(""); + if (realRequest instanceof Request) { + // user efficient header iteration + for (HttpField field : ((Request) realRequest).getHttpFields()) { + builderHeader(upReqBuilder, field.getName(), field.getValue()); + } + } else { + // slower iteration used for test case fake request only + for (String name : Collections.list(realRequest.getHeaderNames())) { + String value = realRequest.getHeader(name); + builderHeader(upReqBuilder, name, value); + } + } + + AppinfoPb.Handler handler = + upReqBuilder + .getHandler() + .newBuilderForType() + .setType(AppinfoPb.Handler.HANDLERTYPE.CGI_BIN.getNumber()) + .setPath("unused") + .build(); + upReqBuilder.setHandler(handler); + + HttpPb.HttpRequest.Builder httpRequest = + upReqBuilder + .getRequestBuilder() + .setHttpVersion(realRequest.getProtocol()) + .setProtocol(realRequest.getMethod()) + .setUrl(getUrl(realRequest)) + .setUserIp(realRequest.getRemoteAddr()); + + if (realRequest instanceof Request) { + // user efficient header iteration + for (HttpField field : ((Request) realRequest).getHttpFields()) { + requestHeader(upReqBuilder, httpRequest, field.getName(), field.getValue()); + } + } else { + // slower iteration used for test case fake request only + for (String name : Collections.list(realRequest.getHeaderNames())) { + String value = realRequest.getHeader(name); + requestHeader(upReqBuilder, httpRequest, name, value); + } + } + + try { + httpRequest.setPostdata(ByteString.readFrom(realRequest.getInputStream())); + } catch (IOException ex) { + throw new IllegalStateException("Could not read POST content:", ex); + } + + if ("/_ah/background".equals(realRequest.getRequestURI())) { + if (WARMUP_IP.equals(httpRequest.getUserIp())) { + upReqBuilder.setRequestType(UPRequest.RequestType.BACKGROUND); + } + } else if ("/_ah/start".equals(realRequest.getRequestURI())) { + if (WARMUP_IP.equals(httpRequest.getUserIp())) { + // This request came from within App Engine via secure internal channels; tell Jetty + // it's HTTPS to avoid 403 because of web.xml security-constraint checks. + httpRequest.setIsHttps(true); + } + } + + return upReqBuilder.build(); + } + + private static void builderHeader(UPRequest.Builder upReqBuilder, String name, String value) { + if (Strings.isNullOrEmpty(value)) { + return; + } + String lower = Ascii.toLowerCase(name); + switch (lower) { + case X_APPENGINE_API_TICKET: + upReqBuilder.setSecurityTicket(value); + return; + + case X_APPENGINE_USER_EMAIL: + upReqBuilder.setEmail(value); + return; + + case X_APPENGINE_USER_NICKNAME: + upReqBuilder.setNickname(value); + return; + + case X_APPENGINE_USER_IS_ADMIN: + upReqBuilder.setIsAdmin(value.equals(IS_ADMIN_HEADER_VALUE)); + return; + + case X_APPENGINE_AUTH_DOMAIN: + upReqBuilder.setAuthDomain(value); + return; + + case X_APPENGINE_USER_ORGANIZATION: + upReqBuilder.setUserOrganization(value); + return; + + case X_APPENGINE_LOAS_PEER_USERNAME: + upReqBuilder.setPeerUsername(value); + return; + + case X_APPENGINE_GAIA_ID: + upReqBuilder.setGaiaId(Long.parseLong(value)); + return; + + case X_APPENGINE_GAIA_AUTHUSER: + upReqBuilder.setAuthuser(value); + return; + + case X_APPENGINE_GAIA_SESSION: + upReqBuilder.setGaiaSession(value); + return; + + case X_APPENGINE_APPSERVER_DATACENTER: + upReqBuilder.setAppserverDatacenter(value); + return; + + case X_APPENGINE_APPSERVER_TASK_BNS: + upReqBuilder.setAppserverTaskBns(value); + return; + + case X_APPENGINE_USER_ID: + upReqBuilder.setObfuscatedGaiaId(value); + return; + + case X_APPENGINE_DEFAULT_VERSION_HOSTNAME: + upReqBuilder.setDefaultVersionHostname(value); + return; + + case X_APPENGINE_REQUEST_LOG_ID: + upReqBuilder.setRequestLogId(value); + return; + + default: + return; + } + } + + private void requestHeader( + UPRequest.Builder upReqBuilder, HttpRequest.Builder httpRequest, String name, String value) { + if (Strings.isNullOrEmpty(value)) { + return; + } + String lower = Ascii.toLowerCase(name); + switch (lower) { + case X_APPENGINE_TRUSTED_IP_REQUEST: + // If there is a value, then the application is trusted + // If the value is IS_TRUSTED, then the user is trusted + httpRequest.setTrusted(value.equals(IS_TRUSTED)); + upReqBuilder.setIsTrustedApp(true); + break; + + case X_APPENGINE_HTTPS: + httpRequest.setIsHttps(value.equals("on")); + break; + + case X_APPENGINE_USER_IP: + httpRequest.setUserIp(value); + break; + + case X_FORWARDED_PROTO: + httpRequest.setIsHttps(value.equals("https")); + break; + + case X_CLOUD_TRACE_CONTEXT: + try { + TraceContextProto proto = TraceContextHelper.parseTraceContextHeader(value); + upReqBuilder.setTraceContext(proto); + } catch (NumberFormatException e) { + logger.atWarning().withCause(e).log("Could not parse trace context header: %s", value); + } + break; + + case X_GOOGLE_INTERNAL_SKIPADMINCHECK: + // may be set by X_APPENGINE_QUEUENAME below + if (upReqBuilder.getRuntimeHeadersList().stream() + .map(ParsedHttpHeader::getKey) + .noneMatch(X_GOOGLE_INTERNAL_SKIPADMINCHECK_UC::equalsIgnoreCase)) { + upReqBuilder.addRuntimeHeaders( + createRuntimeHeader(X_GOOGLE_INTERNAL_SKIPADMINCHECK_UC, "true")); + } + break; + + case X_APPENGINE_QUEUENAME: + httpRequest.setIsOffline(true); + // See b/139183416, allow for cron jobs and task queues to access login: admin urls + if (upReqBuilder.getRuntimeHeadersList().stream() + .map(ParsedHttpHeader::getKey) + .noneMatch(X_GOOGLE_INTERNAL_SKIPADMINCHECK_UC::equalsIgnoreCase)) { + upReqBuilder.addRuntimeHeaders( + createRuntimeHeader(X_GOOGLE_INTERNAL_SKIPADMINCHECK_UC, "true")); + } + break; + + case X_APPENGINE_TIMEOUT_MS: + upReqBuilder.addRuntimeHeaders(createRuntimeHeader(X_APPENGINE_TIMEOUT_MS, value)); + break; + + case X_GOOGLE_INTERNAL_PROFILER: + try { + TextFormat.merge(value, upReqBuilder.getProfilerSettingsBuilder()); + } catch (IOException ex) { + throw new IllegalStateException("X-Google-Internal-Profiler read content error:", ex); + } + break; + + default: + break; + } + if (passThroughPrivateHeaders || !PRIVATE_APPENGINE_HEADERS.contains(lower)) { + // Only non AppEngine specific headers are passed to the application. + httpRequest.addHeadersBuilder().setKey(name).setValue(value); + } + } + + private String getUrl(HttpServletRequest req) { + StringBuffer url = req.getRequestURL(); + String query = req.getQueryString(); + // No need to escape, URL retains any %-escaping it might have, which is what we want. + if (query != null) { + url.append('?').append(query); + } + return url.toString(); + } + + /** + * Populates a response object from some error message. + * + * @param resp response message to fill with info + * @param errMsg error text. + */ + static void populateErrorResponse(HttpServletResponse resp, String errMsg) { + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + try { + ServletOutputStream outstr = resp.getOutputStream(); + outstr.print("Server Error"); + outstr.print("" + HtmlEscapers.htmlEscaper().escape(errMsg) + ""); + } catch (IOException iox) { + throw new IllegalStateException(iox); + } + } + + private static HttpPb.ParsedHttpHeader.Builder createRuntimeHeader(String key, String value) { + return HttpPb.ParsedHttpHeader.newBuilder().setKey(key).setValue(value); + } +} From 75c5b2ff4e3ba284e3ff54f61bf8d8a5258ca416 Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Mon, 17 Oct 2022 09:10:06 -0700 Subject: [PATCH 02/50] Add test for UPRequest and UPResponse translation PiperOrigin-RevId: 481656085 Change-Id: I28ecaae64d5cb676b60139b72d66d64f534e92a0 --- .../jetty94/UPRequestTranslatorTest.java | 478 ++++++++++++++++++ 1 file changed, 478 insertions(+) create mode 100644 runtime/impl/src/test/java/com/google/apphosting/runtime/jetty94/UPRequestTranslatorTest.java diff --git a/runtime/impl/src/test/java/com/google/apphosting/runtime/jetty94/UPRequestTranslatorTest.java b/runtime/impl/src/test/java/com/google/apphosting/runtime/jetty94/UPRequestTranslatorTest.java new file mode 100644 index 000000000..92fff79f7 --- /dev/null +++ b/runtime/impl/src/test/java/com/google/apphosting/runtime/jetty94/UPRequestTranslatorTest.java @@ -0,0 +1,478 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty94; + +import static com.google.common.base.Strings.nullToEmpty; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.toSet; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.apphosting.base.protos.HttpPb; +import com.google.apphosting.base.protos.HttpPb.ParsedHttpHeader; +import com.google.apphosting.base.protos.RuntimePb; +import com.google.apphosting.base.protos.TraceId.TraceIdProto; +import com.google.apphosting.base.protos.TracePb.TraceContextProto; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.ExtensionRegistry; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import javax.servlet.ReadListener; +import javax.servlet.ServletInputStream; +import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.stubbing.Answer; + +@RunWith(JUnit4.class) +public final class UPRequestTranslatorTest { + private static final String X_APPENGINE_HTTPS = "X-AppEngine-Https"; + private static final String X_APPENGINE_USER_IP = "X-AppEngine-User-IP"; + private static final String X_APPENGINE_USER_EMAIL = "X-AppEngine-User-Email"; + private static final String X_APPENGINE_AUTH_DOMAIN = "X-AppEngine-Auth-Domain"; + private static final String X_APPENGINE_USER_ID = "X-AppEngine-User-Id"; + private static final String X_APPENGINE_USER_NICKNAME = "X-AppEngine-User-Nickname"; + private static final String X_APPENGINE_USER_ORGANIZATION = "X-AppEngine-User-Organization"; + private static final String X_APPENGINE_USER_IS_ADMIN = "X-AppEngine-User-Is-Admin"; + private static final String X_APPENGINE_TRUSTED_IP_REQUEST = "X-AppEngine-Trusted-IP-Request"; + private static final String X_APPENGINE_LOAS_PEER_USERNAME = "X-AppEngine-LOAS-Peer-Username"; + private static final String X_APPENGINE_GAIA_ID = "X-AppEngine-Gaia-Id"; + private static final String X_APPENGINE_GAIA_AUTHUSER = "X-AppEngine-Gaia-Authuser"; + private static final String X_APPENGINE_GAIA_SESSION = "X-AppEngine-Gaia-Session"; + private static final String X_APPENGINE_APPSERVER_DATACENTER = "X-AppEngine-Appserver-Datacenter"; + private static final String X_APPENGINE_APPSERVER_TASK_BNS = "X-AppEngine-Appserver-Task-Bns"; + private static final String X_APPENGINE_DEFAULT_VERSION_HOSTNAME = + "X-AppEngine-Default-Version-Hostname"; + private static final String X_APPENGINE_REQUEST_LOG_ID = "X-AppEngine-Request-Log-Id"; + private static final String X_APPENGINE_QUEUENAME = "X-AppEngine-QueueName"; + private static final String X_GOOGLE_INTERNAL_SKIPADMINCHECK = "X-Google-Internal-SkipAdminCheck"; + private static final String X_CLOUD_TRACE_CONTEXT = "X-Cloud-Trace-Context"; + private static final String X_APPENGINE_TIMEOUT_MS = "X-AppEngine-Timeout-Ms"; + + UPRequestTranslator translator; + + @Before + public void setUp() throws Exception { + ImmutableMap fakeEnv = + ImmutableMap.of( + "GAE_VERSION", "3.14", + "GOOGLE_CLOUD_PROJECT", "mytestappid", + "GAE_APPLICATION", "s~mytestappid", + "GAE_SERVICE", "mytestservice"); + + translator = + new UPRequestTranslator(new AppInfoFactory(fakeEnv), /*passThroughPrivateHeaders=*/ false); + } + + @Test + public void translateWithoutAppEngineHeaders() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", + "127.0.0.1", + ImmutableMap.of("testheader", "testvalue")); + + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + + HttpPb.HttpRequest httpRequestPb = translatedUpRequest.getRequest(); + assertThat(httpRequestPb.getHttpVersion()).isEqualTo("HTTP/1.0"); + assertThat(httpRequestPb.getIsHttps()).isFalse(); + assertThat(httpRequestPb.getProtocol()).isEqualTo("GET"); + assertThat(httpRequestPb.getUserIp()).isEqualTo("127.0.0.1"); + assertThat(httpRequestPb.getIsOffline()).isFalse(); + assertThat(httpRequestPb.getUrl()).isEqualTo("http://myapp.appspot.com:80/foo/bar?a=b"); + assertThat(httpRequestPb.getHeadersList()).hasSize(2); + for (ParsedHttpHeader header : httpRequestPb.getHeadersList()) { + assertThat(header.getKey()).isAnyOf("testheader", "host"); + assertThat(header.getValue()).isAnyOf("testvalue", "myapp.appspot.com"); + } + + assertThat(translatedUpRequest.getAppId()).isEqualTo("s~mytestappid"); + assertThat(translatedUpRequest.getVersionId()).isEqualTo("mytestservice:3.14"); + assertThat(translatedUpRequest.getModuleId()).isEqualTo("mytestservice"); + assertThat(translatedUpRequest.getModuleVersionId()).isEqualTo("3.14"); + assertThat(translatedUpRequest.getSecurityTicket()).isEqualTo("secretkey"); + assertThat(translatedUpRequest.getNickname()).isEmpty(); + assertThat(translatedUpRequest.getEmail()).isEmpty(); + assertThat(translatedUpRequest.getUserOrganization()).isEmpty(); + assertThat(translatedUpRequest.getIsAdmin()).isFalse(); + assertThat(translatedUpRequest.getPeerUsername()).isEmpty(); + assertThat(translatedUpRequest.getAppserverDatacenter()).isEmpty(); + assertThat(translatedUpRequest.getAppserverTaskBns()).isEmpty(); + } + + private static final ImmutableMap BASE_APPENGINE_HEADERS = + ImmutableMap.builder() + .put(X_APPENGINE_USER_NICKNAME, "anickname") + .put(X_APPENGINE_USER_IP, "auserip") + .put(X_APPENGINE_USER_EMAIL, "ausermail") + .put(X_APPENGINE_AUTH_DOMAIN, "aauthdomain") + .put(X_APPENGINE_USER_ID, "auserid") + .put(X_APPENGINE_USER_ORGANIZATION, "auserorg") + .put(X_APPENGINE_USER_IS_ADMIN, "false") + .put(X_APPENGINE_TRUSTED_IP_REQUEST, "atrustedip") + .put(X_APPENGINE_LOAS_PEER_USERNAME, "aloasname") + .put(X_APPENGINE_GAIA_ID, "3142406") + .put(X_APPENGINE_GAIA_AUTHUSER, "aauthuser") + .put(X_APPENGINE_GAIA_SESSION, "agaiasession") + .put(X_APPENGINE_APPSERVER_DATACENTER, "adatacenter") + .put(X_APPENGINE_APPSERVER_TASK_BNS, "ataskbns") + .put(X_APPENGINE_HTTPS, "on") + .put(X_APPENGINE_DEFAULT_VERSION_HOSTNAME, "foo.appspot.com") + .put(X_APPENGINE_REQUEST_LOG_ID, "logid") + .put(X_APPENGINE_TIMEOUT_MS, "20000") + .buildOrThrow(); + + @Test + public void translateWithAppEngineHeaders() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", "127.0.0.1", BASE_APPENGINE_HEADERS); + + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + + HttpPb.HttpRequest httpRequestPb = translatedUpRequest.getRequest(); + assertThat(httpRequestPb.getHttpVersion()).isEqualTo("HTTP/1.0"); + assertThat(httpRequestPb.getIsHttps()).isTrue(); + assertThat(httpRequestPb.getProtocol()).isEqualTo("GET"); + assertThat(httpRequestPb.getUserIp()).isEqualTo("auserip"); + assertThat(httpRequestPb.getUrl()).isEqualTo("http://myapp.appspot.com:80/foo/bar?a=b"); + assertThat(httpRequestPb.getTrusted()).isFalse(); + ImmutableSet appengineHeaderNames = + httpRequestPb.getHeadersList().stream() + .map(h -> Ascii.toLowerCase(h.getKey())) + .filter(h -> h.startsWith("x-appengine-")) + .collect(toImmutableSet()); + assertThat(appengineHeaderNames).isEmpty(); + + assertThat(translatedUpRequest.getModuleVersionId()).isEqualTo("3.14"); + assertThat(translatedUpRequest.getSecurityTicket()).isEqualTo("secretkey"); + assertThat(translatedUpRequest.getModuleId()).isEqualTo("mytestservice"); + assertThat(translatedUpRequest.getNickname()).isEqualTo("anickname"); + assertThat(translatedUpRequest.getEmail()).isEqualTo("ausermail"); + assertThat(translatedUpRequest.getUserOrganization()).isEqualTo("auserorg"); + assertThat(translatedUpRequest.getIsAdmin()).isFalse(); + assertThat(translatedUpRequest.getPeerUsername()).isEqualTo("aloasname"); + assertThat(translatedUpRequest.getGaiaId()).isEqualTo(3142406); + assertThat(translatedUpRequest.getAuthuser()).isEqualTo("aauthuser"); + assertThat(translatedUpRequest.getGaiaSession()).isEqualTo("agaiasession"); + assertThat(translatedUpRequest.getAppserverDatacenter()).isEqualTo("adatacenter"); + assertThat(translatedUpRequest.getAppserverTaskBns()).isEqualTo("ataskbns"); + assertThat(translatedUpRequest.getDefaultVersionHostname()).isEqualTo("foo.appspot.com"); + assertThat(translatedUpRequest.getRequestLogId()).isEqualTo("logid"); + assertThat(translatedUpRequest.getRequest().getIsOffline()).isFalse(); + assertThat(translatedUpRequest.getIsTrustedApp()).isTrue(); + ImmutableMap runtimeHeaders = + translatedUpRequest.getRuntimeHeadersList().stream() + .collect(toImmutableMap(h -> Ascii.toLowerCase(h.getKey()), h -> h.getValue())); + assertThat(runtimeHeaders) + .doesNotContainKey(Ascii.toLowerCase(X_GOOGLE_INTERNAL_SKIPADMINCHECK)); + assertThat(runtimeHeaders).containsEntry(Ascii.toLowerCase(X_APPENGINE_TIMEOUT_MS), "20000"); + } + + @Test + public void translateWithAppEngineHeadersIncludingQueueName() throws Exception { + ImmutableMap appengineHeaders = + ImmutableMap.builder() + .putAll(BASE_APPENGINE_HEADERS) + .put(X_APPENGINE_QUEUENAME, "default") + .buildOrThrow(); + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", "127.0.0.1", appengineHeaders); + + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + HttpPb.HttpRequest httpRequestPb = translatedUpRequest.getRequest(); + ImmutableSet appengineHeaderNames = + httpRequestPb.getHeadersList().stream() + .map(h -> Ascii.toLowerCase(h.getKey())) + .filter(h -> h.startsWith("x-appengine-")) + .collect(toImmutableSet()); + assertThat(appengineHeaderNames).containsExactly(Ascii.toLowerCase(X_APPENGINE_QUEUENAME)); + ImmutableMap runtimeHeaders = + translatedUpRequest.getRuntimeHeadersList().stream() + .collect(toImmutableMap(h -> Ascii.toLowerCase(h.getKey()), h -> h.getValue())); + assertThat(runtimeHeaders) + .containsEntry(Ascii.toLowerCase(X_GOOGLE_INTERNAL_SKIPADMINCHECK), "true"); + assertThat(translatedUpRequest.getRequest().getIsOffline()).isTrue(); + } + + @Test + public void translateWithAppEngineHeadersTrustedUser() throws Exception { + // Change the trusted-ip-request header from "atrustedip" to the specific value "1", which means + // that both the app and the user are trusted. + Map appengineHeaders = new HashMap<>(BASE_APPENGINE_HEADERS); + appengineHeaders.put(X_APPENGINE_TRUSTED_IP_REQUEST, "1"); + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", + "127.0.0.1", + ImmutableMap.copyOf(appengineHeaders)); + + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + + HttpPb.HttpRequest httpRequestPb = translatedUpRequest.getRequest(); + assertThat(httpRequestPb.getHttpVersion()).isEqualTo("HTTP/1.0"); + assertThat(httpRequestPb.getIsHttps()).isTrue(); + assertThat(httpRequestPb.getProtocol()).isEqualTo("GET"); + assertThat(httpRequestPb.getUserIp()).isEqualTo("auserip"); + assertThat(httpRequestPb.getUrl()).isEqualTo("http://myapp.appspot.com:80/foo/bar?a=b"); + assertThat(httpRequestPb.getTrusted()).isTrue(); + ImmutableSet appengineHeaderNames = + httpRequestPb.getHeadersList().stream() + .map(h -> Ascii.toLowerCase(h.getKey())) + .filter(h -> h.startsWith("x-appengine-")) + .collect(toImmutableSet()); + assertThat(appengineHeaderNames).isEmpty(); + + assertThat(translatedUpRequest.getModuleVersionId()).isEqualTo("3.14"); + assertThat(translatedUpRequest.getSecurityTicket()).isEqualTo("secretkey"); + assertThat(translatedUpRequest.getModuleId()).isEqualTo("mytestservice"); + assertThat(translatedUpRequest.getNickname()).isEqualTo("anickname"); + assertThat(translatedUpRequest.getEmail()).isEqualTo("ausermail"); + assertThat(translatedUpRequest.getUserOrganization()).isEqualTo("auserorg"); + assertThat(translatedUpRequest.getIsAdmin()).isFalse(); + assertThat(translatedUpRequest.getPeerUsername()).isEqualTo("aloasname"); + assertThat(translatedUpRequest.getGaiaId()).isEqualTo(3142406); + assertThat(translatedUpRequest.getAuthuser()).isEqualTo("aauthuser"); + assertThat(translatedUpRequest.getGaiaSession()).isEqualTo("agaiasession"); + assertThat(translatedUpRequest.getAppserverDatacenter()).isEqualTo("adatacenter"); + assertThat(translatedUpRequest.getAppserverTaskBns()).isEqualTo("ataskbns"); + assertThat(translatedUpRequest.getDefaultVersionHostname()).isEqualTo("foo.appspot.com"); + assertThat(translatedUpRequest.getRequestLogId()).isEqualTo("logid"); + assertThat( + translatedUpRequest.getRuntimeHeadersList().stream() + .map(h -> Ascii.toLowerCase(h.getKey())) + .collect(toSet())) + .doesNotContain(Ascii.toLowerCase(X_GOOGLE_INTERNAL_SKIPADMINCHECK)); + assertThat(translatedUpRequest.getRequest().getIsOffline()).isFalse(); + assertThat(translatedUpRequest.getIsTrustedApp()).isTrue(); + } + + @Test + public void translateEmptyGaiaIdInAppEngineHeaders() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", + "127.0.0.1", + ImmutableMap.of(X_APPENGINE_GAIA_ID, "")); + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + assertThat(translatedUpRequest.getGaiaId()).isEqualTo(0); + } + + @Test + public void translateErrorPageFromHttpResponseError() throws Exception { + HttpServletResponse httpResponse = mock(HttpServletResponse.class); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + when(httpResponse.getOutputStream()).thenReturn(copyingOutputStream(out)); + UPRequestTranslator.populateErrorResponse(httpResponse, "Expected error during test."); + + verify(httpResponse).setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + verify(httpResponse, never()).addHeader(any(), any()); + verify(httpResponse, never()).setHeader(any(), any()); + assertThat(out.toString("UTF-8")) + .isEqualTo( + "Server Error" + + "Expected error during test."); + } + + @Test + public void translateSkipAdminCheckInAppEngineHeaders() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", + "127.0.0.1", + ImmutableMap.of(X_GOOGLE_INTERNAL_SKIPADMINCHECK, "true")); + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + assertThat(translatedUpRequest.getRuntimeHeadersList()) + .contains( + ParsedHttpHeader.newBuilder() + .setKey(X_GOOGLE_INTERNAL_SKIPADMINCHECK) + .setValue("true") + .build()); + } + + @Test + public void translateQueueNameSetsSkipAdminCheckInAppEngineHeaders() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", + "127.0.0.1", + ImmutableMap.of(X_APPENGINE_QUEUENAME, "__cron__")); + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + assertThat(translatedUpRequest.getRuntimeHeadersList()) + .contains( + ParsedHttpHeader.newBuilder() + .setKey(X_GOOGLE_INTERNAL_SKIPADMINCHECK) + .setValue("true") + .build()); + } + + @Test + public void translateBackgroundURISetsBackgroundRequestType() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/_ah/background?a=b", + "127.0.0.1", + ImmutableMap.of(X_APPENGINE_USER_IP, "0.1.0.3")); + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + assertThat(translatedUpRequest.getRequestType()) + .isEqualTo(RuntimePb.UPRequest.RequestType.BACKGROUND); + } + + @Test + public void translateNonBackgroundURIDoesNotSetsBackgroundRequestType() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/foo/bar?a=b", + "127.0.0.1", + ImmutableMap.of(X_APPENGINE_USER_IP, "0.1.0.3")); + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + assertThat(translatedUpRequest.getRequestType()) + .isNotEqualTo(RuntimePb.UPRequest.RequestType.BACKGROUND); + } + + @Test + public void translateRealIpDoesNotSetsBackgroundRequestType() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/_ah/background?a=b", + "127.0.0.1", + ImmutableMap.of(X_APPENGINE_USER_IP, "1.2.3.4")); + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + assertThat(translatedUpRequest.getRequestType()) + .isNotEqualTo(RuntimePb.UPRequest.RequestType.BACKGROUND); + } + + @Test + public void translateCloudContextInAppEngineHeaders() throws Exception { + HttpServletRequest httpRequest = + mockServletRequest( + "http://myapp.appspot.com:80/_ah/background?a=b", + "127.0.0.1", + ImmutableMap.of(X_CLOUD_TRACE_CONTEXT, "000000000000007b00000000000001c8/789;o=1")); + RuntimePb.UPRequest translatedUpRequest = translator.translateRequest(httpRequest); + TraceContextProto contextProto = translatedUpRequest.getTraceContext(); + TraceIdProto traceIdProto = + TraceIdProto.parseFrom(contextProto.getTraceId(), ExtensionRegistry.getEmptyRegistry()); + String traceIdString = String.format("%016x%016x", traceIdProto.getHi(), traceIdProto.getLo()); + assertThat(traceIdString).isEqualTo("000000000000007b00000000000001c8"); + assertThat(contextProto.getSpanId()).isEqualTo(789L); + assertThat(contextProto.getTraceMask()).isEqualTo(1L); + } + + private static HttpServletRequest mockServletRequest( + String url, String remoteAddr, ImmutableMap userHeaders) { + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + String urlWithoutQuery = + uri.getScheme() + + "://" + + uri.getHost() + + (uri.getPort() > 0 ? (":" + uri.getPort()) : "") + + nullToEmpty(uri.getPath()); + ImmutableMap headers = + ImmutableMap.builder() + .putAll(userHeaders) + .put("host", uri.getHost()) + .buildOrThrow(); + HttpServletRequest httpRequest = mock(HttpServletRequest.class); + when(httpRequest.getProtocol()).thenReturn("HTTP/1.0"); + when(httpRequest.getMethod()).thenReturn("GET"); + @SuppressWarnings("JdkObsolete") // imposed by the Servlet API + Answer requestUrlAnswer = invocation -> new StringBuffer(urlWithoutQuery); + when(httpRequest.getRequestURL()).thenAnswer(requestUrlAnswer); + when(httpRequest.getRequestURI()).thenReturn(uri.getPath()); + when(httpRequest.getQueryString()).thenReturn(uri.getQuery()); + when(httpRequest.getRemoteAddr()).thenReturn(remoteAddr); + when(httpRequest.getHeaderNames()) + .thenAnswer(invocation -> Collections.enumeration(headers.keySet())); + headers.forEach((k, v) -> when(httpRequest.getHeader(k)).thenReturn(v)); + try { + when(httpRequest.getInputStream()).thenReturn(emptyInputStream()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return httpRequest; + } + + private static ServletInputStream emptyInputStream() { + return new ServletInputStream() { + @Override + public int read() { + return -1; + } + + @Override + public void setReadListener(ReadListener listener) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isReady() { + return true; + } + + @Override + public boolean isFinished() { + return true; + } + }; + } + + private static ServletOutputStream copyingOutputStream(OutputStream out) { + return new ServletOutputStream() { + @Override + public void write(int b) throws IOException { + out.write(b); + } + + @Override + public void setWriteListener(WriteListener listener) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isReady() { + return true; + } + }; + } +} From dfee1bac62cf94e6104138d20a6b7a6197b07657 Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Mon, 17 Oct 2022 09:29:10 -0700 Subject: [PATCH 03/50] Make JettyServerConnectorWithReusePort public PiperOrigin-RevId: 481660628 Change-Id: Idd9093854dc2c99c16a8dac0bcfac0307356085f --- .../runtime/jetty94/JettyServerConnectorWithReusePort.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServerConnectorWithReusePort.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServerConnectorWithReusePort.java index d4f95dadb..ef546a938 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServerConnectorWithReusePort.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServerConnectorWithReusePort.java @@ -33,7 +33,7 @@ * A wrapper for Jetty to add support for SO_REUSEPORT. (Jetty 9.x does not directly expose it as a * setting.) SO_REUSEPORT only works when running with a Java 9+ JDK. */ -class JettyServerConnectorWithReusePort extends ServerConnector { +public class JettyServerConnectorWithReusePort extends ServerConnector { private final boolean reusePort; From a5194c5ed086119c3c6fb174d675f3774e94ecaa Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Tue, 18 Oct 2022 08:50:23 -0700 Subject: [PATCH 04/50] Fix GAE LogQuery API javadoc. PiperOrigin-RevId: 481932274 Change-Id: I908350a6922fa8be57e67ab74b82d88e7ba67ae0 --- .../java/com/google/appengine/api/log/LogQuery.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/com/google/appengine/api/log/LogQuery.java b/api/src/main/java/com/google/appengine/api/log/LogQuery.java index d47ef10c7..3387d3f3c 100644 --- a/api/src/main/java/com/google/appengine/api/log/LogQuery.java +++ b/api/src/main/java/com/google/appengine/api/log/LogQuery.java @@ -293,11 +293,11 @@ public static LogQuery withStartTimeUsec(long startTimeUsec) { } /** - * Create a {@link LogQuery} with the given end time. - * Shorthand for LogQuery.Builder.withDefaults().endTimeMillis(endTimeMillis);. - * Please read the {@link LogQuery} class javadoc for an explanation of - * how end time is used. - * @param endTimeMillis the start time to use, in milliseconds. + * Create a {@link LogQuery} with the given end time. Shorthand for + * LogQuery.Builder.withDefaults().endTimeMillis(endTimeMillis);. Please read the {@link + * LogQuery} class javadoc for an explanation of how end time is used. + * + * @param endTimeMillis the end time to use, in milliseconds. * @return The newly created LogQuery instance. */ public static LogQuery withEndTimeMillis(long endTimeMillis) { From 9ae1eff043f9790046095aeb58961278c84f51c4 Mon Sep 17 00:00:00 2001 From: Liam Miller-Cushon Date: Tue, 18 Oct 2022 19:51:28 -0700 Subject: [PATCH 05/50] Lazily initialize `threadStop0` PiperOrigin-RevId: 482086135 Change-Id: I7c90d32beac68fbba4323d20a899c0fcad4cc89a --- .../apphosting/runtime/RequestManager.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java index 70c378ef4..3aa730bbc 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/RequestManager.java @@ -500,14 +500,15 @@ public void sendDeadline(String securityTicket, boolean isUncatchable) { // So at least for the time being we can still achieve the effect of Thread.stop(Throwable) by // calling the JNI method. That means we don't get the permission checks and so on that come // with Thread.stop, but the code that's calling it is privileged anyway. - private static final Method threadStop0; - - static { - try { - threadStop0 = Thread.class.getDeclaredMethod("stop0", Object.class); - threadStop0.setAccessible(true); - } catch (NoSuchMethodException e) { - throw new RuntimeException(e); + private static class ThreadStop0Holder { + private static final Method threadStop0; + static { + try { + threadStop0 = Thread.class.getDeclaredMethod("stop0", Object.class); + threadStop0.setAccessible(true); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } } } @@ -611,7 +612,7 @@ public void sendDeadline(RequestToken token, boolean isUncatchable) { AccessController.doPrivileged( (PrivilegedAction) () -> { try { - threadStop0.invoke(targetThread, throwable); + ThreadStop0Holder.threadStop0.invoke(targetThread, throwable); } catch (Exception e) { logger.atWarning().withCause(e).log("Failed to stop thread"); } From 60acdc1d461f004423d9cd7bc95f3245201f7e7d Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Wed, 19 Oct 2022 10:54:22 -0700 Subject: [PATCH 06/50] Remove obsolete (default is now to use jars from Maven builds) use.mavenjars flag. PiperOrigin-RevId: 482248400 Change-Id: Idc314e732e945bd50a02286f125e3944e6ed3dfd --- .../java/com/google/apphosting/runtime/JavaRuntimeMain.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java index f47bfd23c..e7854c741 100644 --- a/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java +++ b/runtime/main/src/main/java/com/google/apphosting/runtime/JavaRuntimeMain.java @@ -40,7 +40,11 @@ public class JavaRuntimeMain { private static final Logger logger = Logger.getLogger(JavaRuntimeMain.class.getName()); private static final String PROPERTIES_LOCATION = "WEB-INF/appengine_optional.properties"; - /** This property will be used in ClassPathUtils processing to determine the correct classpath. */ + /** + * This property will be used in ClassPathUtils processing to determine the correct classpath. + * Property must now be true for the Java8 runtime, and is ignored for Java11/17 runtimes which + * can only use maven jars. + */ private static final String USE_MAVEN_JARS = "use.mavenjars"; /** From 705494be6da9bc73a535ba78d48e5f9fa2877c29 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 20 Oct 2022 08:36:13 -0700 Subject: [PATCH 07/50] Add JDK19 as a build/test env for the GAE runtime. PiperOrigin-RevId: 482498764 Change-Id: I9accbdd9e5be050760940d14ba1144f633c03f46 --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 19b351f7c..bd83da22a 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -31,7 +31,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - java: [8, 11, 17] + java: [8, 11, 17, 19] jdk: [temurin] fail-fast: false From c2ddf02bc51980c4a60bae58d924b1b7c101b491 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 20 Oct 2022 10:47:13 -0700 Subject: [PATCH 08/50] Allows new JDK19 compilation/testing for GAE runtime PiperOrigin-RevId: 482535573 Change-Id: I47caa30259a9058004f6e3d738436fc35f68ea68 --- .../api/images/dev/LocalImagesServiceTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/api_dev/src/test/java/com/google/appengine/api/images/dev/LocalImagesServiceTest.java b/api_dev/src/test/java/com/google/appengine/api/images/dev/LocalImagesServiceTest.java index 58484f6b5..ec266daf5 100644 --- a/api_dev/src/test/java/com/google/appengine/api/images/dev/LocalImagesServiceTest.java +++ b/api_dev/src/test/java/com/google/appengine/api/images/dev/LocalImagesServiceTest.java @@ -487,12 +487,14 @@ private void compareImage(String filename, byte[] responseImage) throws IOExcept assertThat(responseImage).isEqualTo(expectedImage); } - private static boolean isJdk11or17() { - return StandardSystemProperty.JAVA_SPECIFICATION_VERSION.value().equals("11") - || StandardSystemProperty.JAVA_SPECIFICATION_VERSION.value().equals("17"); + private static boolean isJDK8() { + return StandardSystemProperty.JAVA_SPECIFICATION_VERSION.value().equals("1.8"); } - private static final String jdk11or17Name(String filename) { + /** + * We have 2 test files per image: one for jdk8 and the other one (11) used by jdk 11,17 and above + */ + private static final String jdk11(String filename) { return filename.replaceAll("(?!-jdk11)\\.(png|jpg)$", "-jdk11.$1"); } @@ -504,8 +506,8 @@ private static final String jdk11or17Name(String filename) { */ private byte[] readImage(String filename) throws IOException { URL resource = null; - if (isJdk11or17()) { - String jdk11Name = jdk11or17Name(filename); + if (!isJDK8()) { + String jdk11Name = jdk11(filename); resource = getClass().getResource("testdata/" + jdk11Name); } if (resource == null) { From f0aa39695882bad180c1e3eda322bdac8161aabd Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 20 Oct 2022 13:45:29 -0700 Subject: [PATCH 09/50] Optimize how we consume Maven built jars inside google code repository for production usage. PiperOrigin-RevId: 482586475 Change-Id: I0d29a3d32f903591260ed0f1512b57267c38655d --- kokoro/gcp_ubuntu/build.sh | 46 ++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/kokoro/gcp_ubuntu/build.sh b/kokoro/gcp_ubuntu/build.sh index 8cb883724..428434f11 100644 --- a/kokoro/gcp_ubuntu/build.sh +++ b/kokoro/gcp_ubuntu/build.sh @@ -27,31 +27,39 @@ echo "JAVA_HOME = $JAVA_HOME" ./mvnw -e clean install -# The artifacts under `${KOKORO_ARTIFACTS_DIR}/maven-artifacts` will be uploaded. -mkdir ${KOKORO_ARTIFACTS_DIR}/maven-artifacts +# The artifacts under `${KOKORO_ARTIFACTS_DIR}/maven-artifacts` will be uploaded as a zip file named maven_jars.binary +TMP_STAGING_LOCATION=${KOKORO_ARTIFACTS_DIR}/tmp +PUBLISHED_LOCATION=${KOKORO_ARTIFACTS_DIR}/maven-artifacts +mkdir ${TMP_STAGING_LOCATION} +mkdir ${PUBLISHED_LOCATION} # Remove jars we do not need in google3. ls **/*.jar rm **/target/*sources.jar || true rm **/target/*tests.jar || true # LINT.IfChange -cp api_legacy/target/appengine-api-legacy*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-api-legacy.jar -cp appengine-api-1.0-sdk/target/appengine-api-1.0-sdk*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-api-1.0-sdk.jar -cp appengine-api-stubs/target/appengine-api-stubs*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-api-stubs.jar -cp appengine_testing/target/appengine-testing*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-testing.jar -cp remoteapi/target/appengine-remote-api*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-remote-api.jar -cp appengine_jsr107/target/appengine-jsr107*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-jsr107.jar -cp runtime_shared/target/runtime-shared*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/runtime-shared.jar -cp lib/tools_api/target/appengine-tools-sdk*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-tools-api.jar -cp lib/xml_validator/target/libxmlvalidator*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/libxmlvalidator.jar -cp runtime/impl/target/runtime-impl*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/runtime-impl.jar -cp runtime/local/target/appengine-local-runtime*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-local-runtime.jar -cp runtime/main/target/runtime-main*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/runtime-main.jar -cp local_runtime_shared/target/appengine-local-runtime-shared*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-local-runtime-shared.jar -cp quickstartgenerator/target/quickstartgenerator*.jar ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/quickstartgenerator.jar +cp api_legacy/target/appengine-api-legacy*.jar ${TMP_STAGING_LOCATION}/appengine-api-legacy.jar +cp appengine-api-1.0-sdk/target/appengine-api-1.0-sdk*.jar ${TMP_STAGING_LOCATION}/appengine-api-1.0-sdk.jar +cp appengine-api-stubs/target/appengine-api-stubs*.jar ${TMP_STAGING_LOCATION}/appengine-api-stubs.jar +cp appengine_testing/target/appengine-testing*.jar ${TMP_STAGING_LOCATION}/appengine-testing.jar +cp remoteapi/target/appengine-remote-api*.jar ${TMP_STAGING_LOCATION}/appengine-remote-api.jar +cp appengine_jsr107/target/appengine-jsr107*.jar ${TMP_STAGING_LOCATION}/appengine-jsr107.jar +cp runtime_shared/target/runtime-shared*.jar ${TMP_STAGING_LOCATION}/runtime-shared.jar +cp lib/tools_api/target/appengine-tools-sdk*.jar ${TMP_STAGING_LOCATION}/appengine-tools-api.jar +cp lib/xml_validator/target/libxmlvalidator*.jar ${TMP_STAGING_LOCATION}/libxmlvalidator.jar +cp runtime/impl/target/runtime-impl*.jar ${TMP_STAGING_LOCATION}/runtime-impl.jar +cp runtime/local/target/appengine-local-runtime*.jar ${TMP_STAGING_LOCATION}/appengine-local-runtime.jar +cp runtime/main/target/runtime-main*.jar ${TMP_STAGING_LOCATION}/runtime-main.jar +cp local_runtime_shared/target/appengine-local-runtime-shared*.jar ${TMP_STAGING_LOCATION}/appengine-local-runtime-shared.jar +cp quickstartgenerator/target/quickstartgenerator*.jar ${TMP_STAGING_LOCATION}/quickstartgenerator.jar -cp -rf sdk_assembly/target/appengine-java-sdk ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/ +cp -rf sdk_assembly/target/appengine-java-sdk ${TMP_STAGING_LOCATION}/ # Make binaries executable. -chmod a+x ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/appengine-java-sdk/bin/* +chmod a+x ${TMP_STAGING_LOCATION}/appengine-java-sdk/bin/* # LINT.ThenChange(//depot/google3/third_party/java_src/appengine_standard/check_build.sh) -cp sdk_assembly/target/google_appengine_java_delta*.zip ${KOKORO_ARTIFACTS_DIR}/maven-artifacts/google_appengine_java_delta_from_maven.zip +cp sdk_assembly/target/google_appengine_java_delta*.zip ${TMP_STAGING_LOCATION}/google_appengine_java_delta_from_maven.zip +cd ${TMP_STAGING_LOCATION} +zip -r ${PUBLISHED_LOCATION}/maven_jars.binary . +# cleanup staging area +cd .. +rm -rf ${TMP_STAGING_LOCATION} From bd1ec98fd7234d4c888736fb8f8ecd2bbd6b645a Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Fri, 21 Oct 2022 17:10:21 -0700 Subject: [PATCH 10/50] Bump external dependencies. PiperOrigin-RevId: 482922972 Change-Id: I97ee958c8a942713c0df6e53ee80c0da66081c2d --- pom.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 78eaa7892..1d833254f 100644 --- a/pom.xml +++ b/pom.xml @@ -386,7 +386,7 @@ com.google.cloud.datastore datastore-v1-proto-client - 2.11.1 + 2.11.5 com.google.geometry @@ -454,7 +454,7 @@ com.google.api.grpc proto-google-common-protos - 2.9.2 + 2.9.6 com.google.code.findbugs @@ -626,22 +626,22 @@ io.grpc grpc-api - 1.49.0 + 1.49.2 io.grpc grpc-stub - 1.49.0 + 1.49.2 io.grpc grpc-protobuf - 1.49.0 + 1.49.2 io.grpc grpc-netty - 1.49.0 + 1.49.2 org.apache.tomcat @@ -656,12 +656,12 @@ joda-time joda-time - 2.11.1 + 2.11.2 org.json json - 20220320 + 20220924 commons-codec From c4f2067f337bc98fe92d9c04e52c6a288bc855af Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Mon, 24 Oct 2022 13:30:34 -0700 Subject: [PATCH 11/50] Minor tweaks to JettyServletEngineAdapter and JettyHttpProxy for reusability PiperOrigin-RevId: 483470487 Change-Id: I21e9159b0ef950651f774f026a9a9ca6af0dad1c --- .../jetty94/AppVersionHandlerFactory.java | 13 ++++++++-- .../runtime/jetty94/JettyHttpProxy.java | 5 +++- .../jetty94/JettyServletEngineAdapter.java | 3 ++- .../runtime/jetty94/UPRequestTranslator.java | 24 +++++++++++++++---- .../jetty94/UPRequestTranslatorTest.java | 5 +++- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java index 0c128c0b7..dabcdfb89 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java @@ -101,12 +101,17 @@ public class AppVersionHandlerFactory { private final Server server; private final String serverInfo; private final WebAppContextFactory contextFactory; + private final boolean useJettyErrorPageHandler; public AppVersionHandlerFactory( - Server server, String serverInfo, WebAppContextFactory contextFactory) { + Server server, + String serverInfo, + WebAppContextFactory contextFactory, + boolean useJettyErrorPageHandler) { this.server = server; this.serverInfo = serverInfo; this.contextFactory = contextFactory; + this.useJettyErrorPageHandler = useJettyErrorPageHandler; } /** @@ -146,7 +151,11 @@ private Handler doCreateHandler(AppVersion appVersion) throws ServletException { context.setServer(server); context.setDefaultsDescriptor(WEB_DEFAULTS_XML); context.setClassLoader(appVersion.getClassLoader()); - context.setErrorHandler(new NullErrorHandler()); + if (useJettyErrorPageHandler) { + context.getErrorHandler().setShowStacks(false); + } else { + context.setErrorHandler(new NullErrorHandler()); + } File qswebxml = new File(contextRoot, "WEB-INF/quickstart-web.xml"); if (qswebxml.exists()) { context.setConfigurationClasses(quickstartConfigurationClasses); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java index e66b8dd73..27051f1db 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java @@ -193,7 +193,10 @@ public ForwardingHandler(ServletEngineAdapter.Config runtimeOptions, Map Date: Wed, 26 Oct 2022 16:10:15 -0700 Subject: [PATCH 12/50] Minor tweaks to JettyServletEngineAdapter and JettyHttpProxy for reusability PiperOrigin-RevId: 484097749 Change-Id: Ia6d01dcb0e048304b36e23f15255d66d7f285273 --- .../jetty94/AppVersionHandlerFactory.java | 13 ++-------- .../runtime/jetty94/JettyHttpProxy.java | 5 +--- .../jetty94/JettyServletEngineAdapter.java | 3 +-- .../runtime/jetty94/UPRequestTranslator.java | 24 ++++--------------- .../jetty94/UPRequestTranslatorTest.java | 5 +--- 5 files changed, 10 insertions(+), 40 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java index dabcdfb89..0c128c0b7 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java @@ -101,17 +101,12 @@ public class AppVersionHandlerFactory { private final Server server; private final String serverInfo; private final WebAppContextFactory contextFactory; - private final boolean useJettyErrorPageHandler; public AppVersionHandlerFactory( - Server server, - String serverInfo, - WebAppContextFactory contextFactory, - boolean useJettyErrorPageHandler) { + Server server, String serverInfo, WebAppContextFactory contextFactory) { this.server = server; this.serverInfo = serverInfo; this.contextFactory = contextFactory; - this.useJettyErrorPageHandler = useJettyErrorPageHandler; } /** @@ -151,11 +146,7 @@ private Handler doCreateHandler(AppVersion appVersion) throws ServletException { context.setServer(server); context.setDefaultsDescriptor(WEB_DEFAULTS_XML); context.setClassLoader(appVersion.getClassLoader()); - if (useJettyErrorPageHandler) { - context.getErrorHandler().setShowStacks(false); - } else { - context.setErrorHandler(new NullErrorHandler()); - } + context.setErrorHandler(new NullErrorHandler()); File qswebxml = new File(contextRoot, "WEB-INF/quickstart-web.xml"); if (qswebxml.exists()) { context.setConfigurationClasses(quickstartConfigurationClasses); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java index 27051f1db..e66b8dd73 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java @@ -193,10 +193,7 @@ public ForwardingHandler(ServletEngineAdapter.Config runtimeOptions, Map Date: Thu, 27 Oct 2022 07:54:24 -0700 Subject: [PATCH 13/50] Minor tweaks to JettyServletEngineAdapter and JettyHttpProxy for reusability PiperOrigin-RevId: 484251696 Change-Id: Ic1c5177be7a0ade96f93a8fc9e8db73e26e3ac51 --- .../jetty94/AppVersionHandlerFactory.java | 13 ++++++++-- .../runtime/jetty94/JettyHttpProxy.java | 5 +++- .../jetty94/JettyServletEngineAdapter.java | 3 ++- .../runtime/jetty94/UPRequestTranslator.java | 24 +++++++++++++++---- .../jetty94/UPRequestTranslatorTest.java | 5 +++- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java index 0c128c0b7..dabcdfb89 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java @@ -101,12 +101,17 @@ public class AppVersionHandlerFactory { private final Server server; private final String serverInfo; private final WebAppContextFactory contextFactory; + private final boolean useJettyErrorPageHandler; public AppVersionHandlerFactory( - Server server, String serverInfo, WebAppContextFactory contextFactory) { + Server server, + String serverInfo, + WebAppContextFactory contextFactory, + boolean useJettyErrorPageHandler) { this.server = server; this.serverInfo = serverInfo; this.contextFactory = contextFactory; + this.useJettyErrorPageHandler = useJettyErrorPageHandler; } /** @@ -146,7 +151,11 @@ private Handler doCreateHandler(AppVersion appVersion) throws ServletException { context.setServer(server); context.setDefaultsDescriptor(WEB_DEFAULTS_XML); context.setClassLoader(appVersion.getClassLoader()); - context.setErrorHandler(new NullErrorHandler()); + if (useJettyErrorPageHandler) { + context.getErrorHandler().setShowStacks(false); + } else { + context.setErrorHandler(new NullErrorHandler()); + } File qswebxml = new File(contextRoot, "WEB-INF/quickstart-web.xml"); if (qswebxml.exists()) { context.setConfigurationClasses(quickstartConfigurationClasses); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java index e66b8dd73..27051f1db 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java @@ -193,7 +193,10 @@ public ForwardingHandler(ServletEngineAdapter.Config runtimeOptions, Map Date: Mon, 31 Oct 2022 09:34:32 -0700 Subject: [PATCH 14/50] Minor tweaks to JettyServletEngineAdapter and JettyHttpProxy for reusability PiperOrigin-RevId: 485072718 Change-Id: I32a9c7490b5953ed75230d4437e033eb39dcf15e --- .../jetty94/AppVersionHandlerFactory.java | 13 ++-------- .../runtime/jetty94/JettyHttpProxy.java | 5 +--- .../jetty94/JettyServletEngineAdapter.java | 3 +-- .../runtime/jetty94/UPRequestTranslator.java | 24 ++++--------------- .../jetty94/UPRequestTranslatorTest.java | 5 +--- 5 files changed, 10 insertions(+), 40 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java index dabcdfb89..0c128c0b7 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java @@ -101,17 +101,12 @@ public class AppVersionHandlerFactory { private final Server server; private final String serverInfo; private final WebAppContextFactory contextFactory; - private final boolean useJettyErrorPageHandler; public AppVersionHandlerFactory( - Server server, - String serverInfo, - WebAppContextFactory contextFactory, - boolean useJettyErrorPageHandler) { + Server server, String serverInfo, WebAppContextFactory contextFactory) { this.server = server; this.serverInfo = serverInfo; this.contextFactory = contextFactory; - this.useJettyErrorPageHandler = useJettyErrorPageHandler; } /** @@ -151,11 +146,7 @@ private Handler doCreateHandler(AppVersion appVersion) throws ServletException { context.setServer(server); context.setDefaultsDescriptor(WEB_DEFAULTS_XML); context.setClassLoader(appVersion.getClassLoader()); - if (useJettyErrorPageHandler) { - context.getErrorHandler().setShowStacks(false); - } else { - context.setErrorHandler(new NullErrorHandler()); - } + context.setErrorHandler(new NullErrorHandler()); File qswebxml = new File(contextRoot, "WEB-INF/quickstart-web.xml"); if (qswebxml.exists()) { context.setConfigurationClasses(quickstartConfigurationClasses); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java index 27051f1db..e66b8dd73 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java @@ -193,10 +193,7 @@ public ForwardingHandler(ServletEngineAdapter.Config runtimeOptions, Map Date: Tue, 1 Nov 2022 12:49:07 -0700 Subject: [PATCH 15/50] Bump Maven dependencies. PiperOrigin-RevId: 485392781 Change-Id: Ie6b9fde4c73845ce942df988c66a95b0877638e9 --- applications/proberapp/pom.xml | 1 + pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/applications/proberapp/pom.xml b/applications/proberapp/pom.xml index 58ecaef7d..2ed2ed881 100644 --- a/applications/proberapp/pom.xml +++ b/applications/proberapp/pom.xml @@ -41,6 +41,7 @@ ${project.version} UTF-8 + target/${project.artifactId}-${project.version} diff --git a/pom.xml b/pom.xml index 1d833254f..83244db9e 100644 --- a/pom.xml +++ b/pom.xml @@ -449,7 +449,7 @@ com.google.api.grpc proto-google-cloud-datastore-v1 - 0.102.1 + 0.103.0 com.google.api.grpc @@ -789,7 +789,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.3.0 + 3.4.1 com.github.os72 From 95a5e5d3f95cf15283b624f86098216a8462caa8 Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Wed, 2 Nov 2022 07:50:07 -0700 Subject: [PATCH 16/50] Exposes access token based auth in Remote API & Cloud-Datastore configs. PiperOrigin-RevId: 485593295 Change-Id: If725e956d90b498da314e4253396dcd60eea5d78 --- .../CloudDatastoreRemoteServiceConfig.java | 20 +++++++++---- .../datastore/CloudDatastoreV1ClientImpl.java | 20 +++++++++++-- .../DatastoreServiceGlobalConfig.java | 11 ++++++- .../tools/remoteapi/RemoteApiOptions.java | 29 ++++++++++++++++--- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreRemoteServiceConfig.java b/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreRemoteServiceConfig.java index ea4d127c3..ffe0d5dd7 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreRemoteServiceConfig.java +++ b/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreRemoteServiceConfig.java @@ -106,6 +106,7 @@ private DatastoreServiceGlobalConfig toInternalConfig() { .hostOverride(hostOverride()) .additionalAppIds(additionalAppIdsAsStrings()) .serviceAccount(serviceAccount()) + .accessToken(accessToken()) .privateKey(privateKey()) .useComputeEngineCredential(useComputeEngineCredential()) .installApiProxyEnvironment(installApiProxyEnvironment()) @@ -179,6 +180,8 @@ String appIdString() { abstract @Nullable PrivateKey privateKey(); + abstract @Nullable String accessToken(); + abstract boolean useComputeEngineCredential(); abstract int maxRetries(); @@ -187,8 +190,7 @@ String appIdString() { abstract boolean asyncStackTraceCaptureEnabled(); - @Nullable - ImmutableSet additionalAppIdsAsStrings() { + @Nullable ImmutableSet additionalAppIdsAsStrings() { if (additionalAppIds() == null) { return null; } @@ -253,8 +255,8 @@ public abstract CloudDatastoreRemoteServiceConfig.Builder installApiProxyEnviron * If set to true, always use a Compute Engine credential instead of using the Application * Default Credentials library to construct the credential. * - *

Cannot be combined with a call to {@link #useServiceAccountCredential(String, - * PrivateKey)}. + *

Cannot be combined with a call to {@link #useServiceAccountCredential(String, PrivateKey)} + * or {@link #accessToken(String)}. */ public abstract CloudDatastoreRemoteServiceConfig.Builder useComputeEngineCredential( boolean value); @@ -273,11 +275,19 @@ public abstract CloudDatastoreRemoteServiceConfig.Builder useComputeEngineCreden public abstract CloudDatastoreRemoteServiceConfig.Builder asyncStackTraceCaptureEnabled( boolean value); + /** + * Sets the access token. + * + *

Cannot be combined with a call to {@link #useComputeEngineCredential(boolean)} or {@link + * #useServiceAccountCredential(String, PrivateKey)}. + */ + public abstract CloudDatastoreRemoteServiceConfig.Builder accessToken(String accessToken); /** * Instructs the client to use a service account credential instead of using the Application * Default Credentials library to construct the credential. * - *

Cannot be combined with a call to {@link #useComputeEngineCredential(boolean)}. + *

Cannot be combined with a call to {@link #useComputeEngineCredential(boolean)} or {@link + * #accessToken(String)}. */ public CloudDatastoreRemoteServiceConfig.Builder useServiceAccountCredential( String serviceAccountId, PrivateKey privateKey) { diff --git a/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java b/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java index 6e6d3aa90..5793f2d23 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java +++ b/api/src/main/java/com/google/appengine/api/datastore/CloudDatastoreV1ClientImpl.java @@ -323,6 +323,15 @@ private static Credential getCredential() throws GeneralSecurityException, IOExc return new ComputeCredential( GoogleNetHttpTransport.newTrustedTransport(), GsonFactory.getDefaultInstance()); } + if (DatastoreServiceGlobalConfig.getConfig().accessToken() != null) { + GoogleCredential credential = + getCredentialBuilder() + .build() + .setAccessToken(DatastoreServiceGlobalConfig.getConfig().accessToken()) + .createScoped(DatastoreOptions.SCOPES); + credential.refreshToken(); + return credential; + } return GoogleCredential.getApplicationDefault().createScoped(DatastoreOptions.SCOPES); } @@ -347,10 +356,15 @@ private static void setProjectEndpoint(String projectId, DatastoreOptions.Builde private static GoogleCredential.Builder getServiceAccountCredentialBuilder(String account) throws GeneralSecurityException, IOException { - return new GoogleCredential.Builder() - .setTransport(GoogleNetHttpTransport.newTrustedTransport()) - .setJsonFactory(GsonFactory.getDefaultInstance()) + return getCredentialBuilder() .setServiceAccountId(account) .setServiceAccountScopes(DatastoreOptions.SCOPES); } + + private static GoogleCredential.Builder getCredentialBuilder() + throws GeneralSecurityException, IOException { + return new GoogleCredential.Builder() + .setTransport(GoogleNetHttpTransport.newTrustedTransport()) + .setJsonFactory(GsonFactory.getDefaultInstance()); + } } diff --git a/api/src/main/java/com/google/appengine/api/datastore/DatastoreServiceGlobalConfig.java b/api/src/main/java/com/google/appengine/api/datastore/DatastoreServiceGlobalConfig.java index 2c8cf7e04..8a292c30d 100644 --- a/api/src/main/java/com/google/appengine/api/datastore/DatastoreServiceGlobalConfig.java +++ b/api/src/main/java/com/google/appengine/api/datastore/DatastoreServiceGlobalConfig.java @@ -201,6 +201,9 @@ static Environment getCurrentApiProxyEnvironment() { @Nullable abstract String emulatorHost(); + @Nullable + abstract String accessToken(); + @Nullable abstract String serviceAccount(); @@ -258,6 +261,8 @@ abstract static class Builder { abstract DatastoreServiceGlobalConfig.Builder useApiProxy(boolean value); + abstract DatastoreServiceGlobalConfig.Builder accessToken(String value); + abstract DatastoreServiceGlobalConfig.Builder serviceAccount(String value); abstract DatastoreServiceGlobalConfig.Builder privateKey(PrivateKey value); @@ -301,7 +306,8 @@ DatastoreServiceGlobalConfig build() { checkState( config.additionalAppIds() == null, "Cannot specify additional app IDs when using API proxy."); - + checkState( + config.accessToken() == null, "Cannot specify access token when using API proxy."); checkState( config.serviceAccount() == null, "Cannot specify service account when using API proxy."); @@ -346,6 +352,9 @@ DatastoreServiceGlobalConfig build() { !(config.serviceAccount() != null && config.useComputeEngineCredential()), "Must not provide a service account and at the same time require the use of Compute " + "Engine credentials."); + checkState( + config.accessToken() == null || config.serviceAccount() == null, + "Must not provide both an access token and a service account."); return config; } diff --git a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiOptions.java b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiOptions.java index f9557e120..4953e9172 100644 --- a/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiOptions.java +++ b/remoteapi/src/main/java/com/google/appengine/tools/remoteapi/RemoteApiOptions.java @@ -227,6 +227,24 @@ public RemoteApiOptions useServiceAccountCredential( return this; } + /** + * Use an access token credential. Overrides any previously-provided credentials. + * + * @param accessToken the access token (generally from {@link GoogleCredential#getAccessToken}) + * @return this {@code RemoteApiOptions} instance + */ + public RemoteApiOptions useAccessToken(String accessToken) { + try { + GoogleCredential credential = getCredentialBuilder().build().setAccessToken(accessToken); + credential = credential.createScoped(OAUTH_SCOPES); + credential.refreshToken(); + setOAuthCredential(credential); + } catch (IOException | GeneralSecurityException e) { + throw new RuntimeException("Failed to build access token credential.", e); + } + return this; + } + /** * Use credentials appropriate for talking to the Development Server. Overrides any * previously-provided credentials. @@ -247,13 +265,16 @@ public RemoteApiOptions useDevelopmentServerCredential() { return this; } - private GoogleCredential.Builder getCredentialBuilder(String serviceAccountId) + private GoogleCredential.Builder getCredentialBuilder() throws GeneralSecurityException, IOException { HttpTransport transport = getOrCreateHttpTransportForOAuth(); JsonFactory jsonFactory = GsonFactory.getDefaultInstance(); - return new GoogleCredential.Builder() - .setTransport(transport) - .setJsonFactory(jsonFactory) + return new GoogleCredential.Builder().setTransport(transport).setJsonFactory(jsonFactory); + } + + private GoogleCredential.Builder getCredentialBuilder(String serviceAccountId) + throws GeneralSecurityException, IOException { + return getCredentialBuilder() .setServiceAccountId(serviceAccountId) .setServiceAccountScopes(OAUTH_SCOPES); } From 772d29857fdf0efc3cb68b4e987ca0f54a22388d Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Wed, 2 Nov 2022 19:42:55 -0700 Subject: [PATCH 17/50] Minor tweaks to JettyServletEngineAdapter and JettyHttpProxy for reusability PiperOrigin-RevId: 485756933 Change-Id: Ibe3762c215d0b9a99de8259d8946dfd0386e7603 --- .../jetty94/AppVersionHandlerFactory.java | 13 ++++++++-- .../runtime/jetty94/JettyHttpProxy.java | 5 +++- .../jetty94/JettyServletEngineAdapter.java | 3 ++- .../runtime/jetty94/UPRequestTranslator.java | 24 +++++++++++++++---- .../jetty94/UPRequestTranslatorTest.java | 5 +++- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java index 0c128c0b7..dabcdfb89 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/AppVersionHandlerFactory.java @@ -101,12 +101,17 @@ public class AppVersionHandlerFactory { private final Server server; private final String serverInfo; private final WebAppContextFactory contextFactory; + private final boolean useJettyErrorPageHandler; public AppVersionHandlerFactory( - Server server, String serverInfo, WebAppContextFactory contextFactory) { + Server server, + String serverInfo, + WebAppContextFactory contextFactory, + boolean useJettyErrorPageHandler) { this.server = server; this.serverInfo = serverInfo; this.contextFactory = contextFactory; + this.useJettyErrorPageHandler = useJettyErrorPageHandler; } /** @@ -146,7 +151,11 @@ private Handler doCreateHandler(AppVersion appVersion) throws ServletException { context.setServer(server); context.setDefaultsDescriptor(WEB_DEFAULTS_XML); context.setClassLoader(appVersion.getClassLoader()); - context.setErrorHandler(new NullErrorHandler()); + if (useJettyErrorPageHandler) { + context.getErrorHandler().setShowStacks(false); + } else { + context.setErrorHandler(new NullErrorHandler()); + } File qswebxml = new File(contextRoot, "WEB-INF/quickstart-web.xml"); if (qswebxml.exists()) { context.setConfigurationClasses(quickstartConfigurationClasses); diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java index e66b8dd73..27051f1db 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java @@ -193,7 +193,10 @@ public ForwardingHandler(ServletEngineAdapter.Config runtimeOptions, Map Date: Thu, 3 Nov 2022 13:06:03 -0700 Subject: [PATCH 18/50] Internal change PiperOrigin-RevId: 485951949 Change-Id: I4c1f142df64fbf346689158a4274d67bcf147a16 --- .../appengine_setup/pom.xml | 46 ++ .../appengine/setup/ApiProxyDelegate.java | 408 +++++++++++++++++ .../appengine/setup/ApiProxyEnvironment.java | 429 ++++++++++++++++++ .../google/appengine/setup/AppLogsWriter.java | 294 ++++++++++++ .../setup/LazyApiProxyEnvironment.java | 117 +++++ .../appengine/setup/RequestThreadFactory.java | 92 ++++ .../google/appengine/setup/RuntimeUtils.java | 38 ++ .../com/google/appengine/setup/TimerImpl.java | 31 ++ .../setup/timer/AbstractIntervalTimer.java | 82 ++++ .../google/appengine/setup/timer/Timer.java | 27 ++ .../setup/utils/http/HttpRequest.java | 22 + 11 files changed, 1586 insertions(+) create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java create mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml b/google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml new file mode 100644 index 000000000..fe2db3481 --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + com.google + appengine_setup + 1.0-SNAPSHOT + jar + AppEngine :: appengine_setup + DO NOT USE - Presently in Beta Mode. Library to help setup AppEngine features (bundled services, session management, etc). + + + 11 + 11 + true + + + + com.google.appengine + appengine-api-1.0-sdk + 2.0.8 + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + \ No newline at end of file diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java new file mode 100644 index 000000000..81c6d6eba --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java @@ -0,0 +1,408 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup; + +import com.google.appengine.repackaged.com.google.common.collect.Lists; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.ApiConfig; +import com.google.apphosting.api.ApiProxy.ApiProxyException; +import com.google.apphosting.api.ApiProxy.LogRecord; +import com.google.apphosting.api.ApiProxy.RPCFailedException; +import com.google.apphosting.utils.remoteapi.RemoteApiPb; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Scanner; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.conn.params.ConnManagerPNames; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.PoolingClientConnectionManager; +import org.apache.http.params.BasicHttpParams; +import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.params.HttpParams; +import org.apache.http.protocol.BasicHttpContext; + +/** + * Delegates AppEngine API calls to a local http API proxy. + *

+ *

Instances should be registered using ApiProxy.setDelegate(ApiProxy.Delegate). + */ +public class ApiProxyDelegate implements ApiProxy.Delegate { + + private static final Logger logger = Logger.getLogger(ApiProxyDelegate.class.getName()); + + public static final String RPC_DEADLINE_HEADER = "X-Google-RPC-Service-Deadline"; + public static final String RPC_STUB_ID_HEADER = "X-Google-RPC-Service-Endpoint"; + public static final String RPC_METHOD_HEADER = "X-Google-RPC-Service-Method"; + + public static final String REQUEST_ENDPOINT = "/rpc_http"; // :8089 + public static final String REQUEST_STUB_ID = "app-engine-apis"; + public static final String REQUEST_STUB_METHOD = "/VMRemoteAPI.CallRemoteAPI"; + + protected static final String API_DEADLINE_KEY = + "com.google.apphosting.api.ApiProxy.api_deadline_key"; + + static final int ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS = 1000; + + protected int defaultTimeoutMs; + protected final ExecutorService executor; + + protected final HttpClient httpclient; + + final IdleConnectionMonitorThread monitorThread; + + private static ClientConnectionManager createConnectionManager() { + PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); + connectionManager.setMaxTotal(ApiProxyEnvironment.MAX_CONCURRENT_API_CALLS); + connectionManager.setDefaultMaxPerRoute(ApiProxyEnvironment.MAX_CONCURRENT_API_CALLS); + return connectionManager; + } + + public ApiProxyDelegate() { + this(new DefaultHttpClient(createConnectionManager())); + } + + + ApiProxyDelegate(HttpClient httpclient) { + this.defaultTimeoutMs = 5 * 60 * 1000; + this.executor = Executors.newCachedThreadPool(); + this.httpclient = httpclient; + this.monitorThread = new IdleConnectionMonitorThread(httpclient.getConnectionManager()); + this.monitorThread.start(); + } + + @Override + public byte[] makeSyncCall( + LazyApiProxyEnvironment environment, + String packageName, + String methodName, + byte[] requestData) + throws ApiProxyException { + return makeSyncCallWithTimeout(environment, packageName, methodName, requestData, + defaultTimeoutMs); + } + + private byte[] makeSyncCallWithTimeout( + LazyApiProxyEnvironment environment, + String packageName, + String methodName, + byte[] requestData, + int timeoutMs) + throws ApiProxyException { + return makeApiCall(environment, packageName, methodName, requestData, timeoutMs, false); + } + + private byte[] makeApiCall(LazyApiProxyEnvironment environment, + String packageName, + String methodName, + byte[] requestData, + int timeoutMs, + boolean wasAsync) { + environment.apiCallStarted(RuntimeUtils.MAX_USER_API_CALL_WAIT_MS, wasAsync); + try { + return runSyncCall(environment, packageName, methodName, requestData, timeoutMs); + } finally { + environment.apiCallCompleted(); + } + } + + + protected byte[] runSyncCall(LazyApiProxyEnvironment environment, String packageName, + String methodName, byte[] requestData, int timeoutMs) { + HttpPost request = createRequest(environment, packageName, methodName, requestData, timeoutMs); + try { + BasicHttpContext context = new BasicHttpContext(); + HttpResponse response = httpclient.execute(request, context); + + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + try (Scanner errorStreamScanner = + new Scanner(new BufferedInputStream(response.getEntity().getContent()));) { + logger.info("Error body: " + errorStreamScanner.useDelimiter("\\Z").next()); + throw new RPCFailedException(packageName, methodName); + } + } + try (BufferedInputStream bis = new BufferedInputStream(response.getEntity().getContent())) { + RemoteApiPb.Response remoteResponse = new RemoteApiPb.Response(); + if (!remoteResponse.parseFrom(bis)) { + logger.info( + "HTTP ApiProxy unable to parse response for " + packageName + "." + methodName); + throw new RPCFailedException(packageName, methodName); + } + if (remoteResponse.hasRpcError() || remoteResponse.hasApplicationError()) { + throw convertRemoteError(remoteResponse, packageName, methodName, logger); + } + return remoteResponse.getResponseAsBytes(); + } + } catch (IOException e) { + logger.info( + "HTTP ApiProxy I/O error for " + packageName + "." + methodName + ": " + e.getMessage()); + throw new RPCFailedException(packageName, methodName); + } finally { + request.releaseConnection(); + } + } + + /** + * Create an HTTP post request suitable for sending to the API server. + * + * @param environment The current VMApiProxyEnvironment + * @param packageName The API call package + * @param methodName The API call method + * @param requestData The POST payload. + * @param timeoutMs The timeout for this request + * @return an HttpPost object to send to the API. + */ + static HttpPost createRequest(LazyApiProxyEnvironment environment, String packageName, + String methodName, byte[] requestData, int timeoutMs) { + RemoteApiPb.Request remoteRequest = new RemoteApiPb.Request(); + remoteRequest.setServiceName(packageName); + remoteRequest.setMethod(methodName); + // Commenting below line to validate the use-cases where security ticket may be needed. So far we did not need. + //remoteRequest.setRequestId(environment.getTicket()); + remoteRequest.setRequestAsBytes(requestData); + + HttpPost request = new HttpPost("http://" + environment.getServer() + REQUEST_ENDPOINT); + request.setHeader(RPC_STUB_ID_HEADER, REQUEST_STUB_ID); + request.setHeader(RPC_METHOD_HEADER, REQUEST_STUB_METHOD); + + HttpParams params = new BasicHttpParams(); + params.setLongParameter(ConnManagerPNames.TIMEOUT, + timeoutMs + ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS); + params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, + timeoutMs + ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS); + params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, + timeoutMs + ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS); + + params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, Boolean.TRUE); + params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, Boolean.FALSE); + request.setParams(params); + + Double deadline = (Double) (environment.getAttributes().get(API_DEADLINE_KEY)); + if (deadline == null) { + request.setHeader(RPC_DEADLINE_HEADER, + Double.toString(TimeUnit.SECONDS.convert(timeoutMs, TimeUnit.MILLISECONDS))); + } else { + request.setHeader(RPC_DEADLINE_HEADER, Double.toString(deadline)); + } + + Object dapperHeader = environment.getAttributes() + .get(ApiProxyEnvironment.AttributeMapping.DAPPER_ID.attributeKey); + if (dapperHeader instanceof String) { + request.setHeader( + ApiProxyEnvironment.AttributeMapping.DAPPER_ID.headerKey, (String) dapperHeader); + } + + ByteArrayEntity postPayload = new ByteArrayEntity(remoteRequest.toByteArray(), + ContentType.APPLICATION_OCTET_STREAM); + postPayload.setChunked(false); + request.setEntity(postPayload); + + return request; + } + + /** + * Convert RemoteApiPb.Response errors to the appropriate exception. + *

+ *

The response must have exactly one of the RpcError and ApplicationError fields set. + * + * @param remoteResponse the Response + * @param packageName the name of the API package. + * @param methodName the name of the method within the API package. + * @param logger the Logger used to create log messages. + * @return ApiProxyException + */ + private static ApiProxyException convertRemoteError(RemoteApiPb.Response remoteResponse, + String packageName, String methodName, Logger logger) { + if (remoteResponse.hasRpcError()) { + return convertApiResponseRpcErrorToException( + remoteResponse.getRpcError(), + packageName, + methodName, + logger); + } + + RemoteApiPb.ApplicationError error = remoteResponse.getApplicationError(); + return new ApiProxy.ApplicationException(error.getCode(), error.getDetail()); + } + + /** + * Convert the RemoteApiPb.RpcError to the appropriate exception. + * + * @param rpcError the RemoteApiPb.RpcError. + * @param packageName the name of the API package. + * @param methodName the name of the method within the API package. + * @param logger the Logger used to create log messages. + * @return ApiProxyException + */ + private static ApiProxyException convertApiResponseRpcErrorToException( + RemoteApiPb.RpcError rpcError, String packageName, String methodName, Logger logger) { + + int rpcCode = rpcError.getCode(); + String errorDetail = rpcError.getDetail(); + if (rpcCode > RemoteApiPb.RpcError.ErrorCode.values().length) { + logger.severe("Received unrecognized error code from server: " + rpcError.getCode() + + " details: " + errorDetail); + return new ApiProxy.UnknownException(packageName, methodName); + } + RemoteApiPb.RpcError.ErrorCode errorCode = RemoteApiPb.RpcError.ErrorCode.values()[ + rpcError.getCode()]; + logger.warning("RPC failed : " + errorCode + " : " + errorDetail); + + switch (errorCode) { + case CALL_NOT_FOUND: + return new ApiProxy.CallNotFoundException(packageName, methodName); + case PARSE_ERROR: + return new ApiProxy.ArgumentException(packageName, methodName); + case SECURITY_VIOLATION: + logger.severe("Security violation: invalid request id used!"); + return new ApiProxy.UnknownException(packageName, methodName); + case CAPABILITY_DISABLED: + return new ApiProxy.CapabilityDisabledException( + errorDetail, packageName, methodName); + case OVER_QUOTA: + return new ApiProxy.OverQuotaException(packageName, methodName); + case REQUEST_TOO_LARGE: + return new ApiProxy.RequestTooLargeException(packageName, methodName); + case RESPONSE_TOO_LARGE: + return new ApiProxy.ResponseTooLargeException(packageName, methodName); + case BAD_REQUEST: + return new ApiProxy.ArgumentException(packageName, methodName); + case CANCELLED: + return new ApiProxy.CancelledException(packageName, methodName); + case FEATURE_DISABLED: + return new ApiProxy.FeatureNotEnabledException( + errorDetail, packageName, methodName); + case DEADLINE_EXCEEDED: + return new ApiProxy.ApiDeadlineExceededException(packageName, methodName); + default: + return new ApiProxy.UnknownException(packageName, methodName); + } + } + + private class MakeSyncCall implements Callable { + private final ApiProxyDelegate delegate; + private final LazyApiProxyEnvironment environment; + private final String packageName; + private final String methodName; + private final byte[] requestData; + private final int timeoutMs; + + public MakeSyncCall(ApiProxyDelegate delegate, + LazyApiProxyEnvironment environment, + String packageName, + String methodName, + byte[] requestData, + int timeoutMs) { + this.delegate = delegate; + this.environment = environment; + this.packageName = packageName; + this.methodName = methodName; + this.requestData = requestData; + this.timeoutMs = timeoutMs; + } + + @Override + public byte[] call() throws Exception { + return delegate.makeApiCall(environment, + packageName, + methodName, + requestData, + timeoutMs, + true); + } + } + + @Override + public Future makeAsyncCall( + LazyApiProxyEnvironment environment, + String packageName, + String methodName, + byte[] request, + ApiConfig apiConfig) { + int timeoutMs = defaultTimeoutMs; + if (apiConfig != null && apiConfig.getDeadlineInSeconds() != null) { + timeoutMs = (int) (apiConfig.getDeadlineInSeconds() * 1000); + } + environment.aSyncApiCallAdded(RuntimeUtils.MAX_USER_API_CALL_WAIT_MS); + return executor.submit(new MakeSyncCall(this, environment, packageName, + methodName, request, timeoutMs)); + } + + @Override + public void log(LazyApiProxyEnvironment environment, LogRecord record) { + if (environment != null) { + environment.addLogRecord(record); + } + } + + @Override + public void flushLogs(LazyApiProxyEnvironment environment) { + if (environment != null) { + environment.flushLogs(); + } + } + + @Override + public List getRequestThreads(LazyApiProxyEnvironment environment) { + Object threadFactory = + environment.getAttributes().get(ApiProxyEnvironment.REQUEST_THREAD_FACTORY_ATTR); + if (threadFactory != null && threadFactory instanceof RequestThreadFactory) { + return ((RequestThreadFactory) threadFactory).getRequestThreads(); + } + logger.warning("Got a call to getRequestThreads() but no VmRequestThreadFactory is available"); + return Lists.newLinkedList(); + } + + /** + * Simple connection watchdog verifying that our connections are alive. Any stale connections are + * cleared as well. + */ + class IdleConnectionMonitorThread extends Thread { + + private final ClientConnectionManager connectionManager; + + public IdleConnectionMonitorThread(ClientConnectionManager connectionManager) { + super("IdleApiConnectionMontorThread"); + this.connectionManager = connectionManager; + this.setDaemon(false); + } + + @Override + public void run() { + try { + while (true) { + connectionManager.closeExpiredConnections(); + connectionManager.closeIdleConnections(60, TimeUnit.SECONDS); + Thread.sleep(5000); + } + } catch (InterruptedException ex) { + } + } + } +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java new file mode 100644 index 000000000..789d95f15 --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java @@ -0,0 +1,429 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup; + +import com.google.appengine.setup.timer.Timer; +import com.google.appengine.setup.utils.http.HttpRequest; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.ApiProxyException; +import com.google.apphosting.api.ApiProxy.LogRecord; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +/** + * Implements the ApiProxy environment. + *

+ * Supports instantiation within a request as well as outside the context of a request. + *

+ * Instances should be registered using ApiProxy.setEnvironmentForCurrentThread(Environment). + */ +public class ApiProxyEnvironment implements ApiProxy.Environment { + + static final String GAE_APPLICATION = "GAE_APPLICATION"; + + static final String GAE_SERVICE = "GAE_SERVICE"; + + static final String GAE_VERSION = "GAE_VERSION"; + + static final String GAE_INSTANCE = "GAE_INSTANCE"; + + public static final String TICKET_HEADER = "X-AppEngine-Api-Ticket"; + public static final String EMAIL_HEADER = "X-AppEngine-User-Email"; + public static final String IS_ADMIN_HEADER = "X-AppEngine-User-Is-Admin"; + public static final String AUTH_DOMAIN_HEADER = "X-AppEngine-Auth-Domain"; + + public static final String BACKEND_ID_KEY = "com.google.appengine.backend.id"; + public static final String INSTANCE_ID_KEY = "com.google.appengine.instance.id"; + public static final String REQUEST_THREAD_FACTORY_ATTR = "com.google.appengine.api.ThreadManager.REQUEST_THREAD_FACTORY"; + public static final String BACKGROUND_THREAD_FACTORY_ATTR = "com.google.appengine.api.ThreadManager.BACKGROUND_THREAD_FACTORY"; + public static final String IS_FEDERATED_USER_KEY = "com.google.appengine.api.users.UserService.is_federated_user"; + public static final String IS_TRUSTED_IP_KEY = "com.google.appengine.runtime.is_trusted_ip"; + public static final String IS_TRUSTED_IP_HEADER = "X-AppEngine-Trusted-IP-Request"; + + private static final long DEFAULT_FLUSH_APP_LOGS_EVERY_BYTE_COUNT = 1024 * 1024L; + private static final int MAX_LOG_FLUSH_SECONDS = 60; + private static final int DEFAULT_MAX_LOG_LINE_SIZE = 8 * 1024; + + static final int MAX_CONCURRENT_API_CALLS = 100; + static final int MAX_PENDING_API_CALLS = 1000; + + /** + * Mapping from HTTP header keys to attribute keys. + */ + static enum AttributeMapping { + USER_ID( + "X-AppEngine-User-Id", + "com.google.appengine.api.users.UserService.user_id_key", + "", false), + USER_ORGANIZATION( + "X-AppEngine-User-Organization", + "com.google.appengine.api.users.UserService.user_organization", + "", false), + FEDERATED_IDENTITY( + "X-AppEngine-Federated-Identity", + "com.google.appengine.api.users.UserService.federated_identity", + "", false), + FEDERATED_PROVIDER( + "X-AppEngine-Federated-Provider", + "com.google.appengine.api.users.UserService.federated_authority", + "", false), + DATACENTER( + "X-AppEngine-Datacenter", + "com.google.apphosting.api.ApiProxy.datacenter", + "", false), + REQUEST_ID_HASH( + "X-AppEngine-Request-Id-Hash", + "com.google.apphosting.api.ApiProxy.request_id_hash", + null, false), + REQUEST_LOG_ID( + "X-AppEngine-Request-Log-Id", + "com.google.appengine.runtime.request_log_id", + null, false), + DAPPER_ID("X-Google-DapperTraceInfo", + "com.google.appengine.runtime.dapper_id", + null, false), + DEFAULT_VERSION_HOSTNAME( + "X-AppEngine-Default-Version-Hostname", + "com.google.appengine.runtime.default_version_hostname", + null, false), + DEFAULT_NAMESPACE_HEADER( + "X-AppEngine-Default-Namespace", + "com.google.appengine.api.NamespaceManager.appsNamespace", + null, false), + CURRENT_NAMESPACE_HEADER( + "X-AppEngine-Current-Namespace", + "com.google.appengine.api.NamespaceManager.currentNamespace", + null, false), + LOAS_PEER_USERNAME( + "X-AppEngine-LOAS-Peer-Username", + "com.google.net.base.peer.loas_peer_username", + "", true), + GAIA_ID( + "X-AppEngine-Gaia-Id", + "com.google.appengine.runtime.gaia_id", + "", true), + GAIA_AUTHUSER( + "X-AppEngine-Gaia-Authuser", + "com.google.appengine.runtime.gaia_authuser", + "", true), + GAIA_SESSION( + "X-AppEngine-Gaia-Session", + "com.google.appengine.runtime.gaia_session", + "", true), + APPSERVER_DATACENTER( + "X-AppEngine-Appserver-Datacenter", + "com.google.appengine.runtime.appserver_datacenter", + "", true), + APPSERVER_TASK_BNS( + "X-AppEngine-Appserver-Task-Bns", + "com.google.appengine.runtime.appserver_task_bns", + "", true); + + String headerKey; + String attributeKey; + Object defaultValue; + private boolean trustedAppOnly; + + /** + * Creates a mapping between an incoming request header and the thread local request attribute + * corresponding to that header. + * + * @param headerKey The HTTP header key. + * @param attributeKey The attribute key. + * @param defaultValue The default value to set if the header is missing, or null if no + * attribute should be set when the header is missing. + * @param trustedAppOnly If true the attribute should only be set for trusted apps. + */ + private AttributeMapping( + String headerKey, String attributeKey, Object defaultValue, boolean trustedAppOnly) { + this.headerKey = headerKey; + this.attributeKey = attributeKey; + this.defaultValue = defaultValue; + this.trustedAppOnly = trustedAppOnly; + } + } + + /** + * Helper method to use during the transition from metadata to environment variables. + * + * @param environmentMap the + * @param envKey The name of the environment variable to check first. + * @return If set the environment variable corresponding to envKey, the metadata entry otherwise. + */ + private static String getEnvOrMetadata(Map environmentMap, + String envKey) { + return environmentMap.get(envKey); + } + + public static ApiProxyEnvironment createFromHeaders(Map envMap, + HttpRequest request, + String server, + Timer wallTimer, + Long millisUntilSoftDeadline) { + String appId = getEnvOrMetadata(envMap, GAE_APPLICATION); + String module = getEnvOrMetadata(envMap, GAE_SERVICE); + String majorVersion = getEnvOrMetadata(envMap, GAE_VERSION); + String instance = getEnvOrMetadata(envMap, GAE_INSTANCE); + String ticket = request.getHeader(TICKET_HEADER); + String email = request.getHeader(EMAIL_HEADER); + boolean admin = false; + String value = request.getHeader(IS_ADMIN_HEADER); + if (value != null && !value.trim().isEmpty()) { + try { + admin = Integer.parseInt(value.trim()) != 0; + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } + String authDomain = request.getHeader(AUTH_DOMAIN_HEADER); + boolean trustedApp = request.getHeader(IS_TRUSTED_IP_HEADER) != null; + + Map attributes = new HashMap<>(); + for (AttributeMapping mapping : AttributeMapping.values()) { + if (mapping.trustedAppOnly && !trustedApp) { + continue; + } + String headerValue = request.getHeader(mapping.headerKey); + if (headerValue != null) { + attributes.put(mapping.attributeKey, headerValue); + } else if (mapping.defaultValue != null) { + attributes.put(mapping.attributeKey, mapping.defaultValue); + } + } + + boolean federatedId = request.getHeader(AttributeMapping.FEDERATED_IDENTITY.headerKey) != null; + attributes.put(IS_FEDERATED_USER_KEY, federatedId); + + attributes.put(BACKEND_ID_KEY, module); + attributes.put(INSTANCE_ID_KEY, instance); + + if (trustedApp) { + boolean trustedIp = "1".equals(request.getHeader(IS_TRUSTED_IP_HEADER)); + attributes.put(IS_TRUSTED_IP_KEY, trustedIp); + } + + ApiProxyEnvironment requestEnvironment = new ApiProxyEnvironment(server, ticket, appId, + module, majorVersion, instance, email, admin, authDomain, + wallTimer, millisUntilSoftDeadline, attributes); + attributes.put(REQUEST_THREAD_FACTORY_ATTR, new RequestThreadFactory(requestEnvironment)); + attributes.put(BACKGROUND_THREAD_FACTORY_ATTR, Executors.defaultThreadFactory()); + + return requestEnvironment; + } + + private final String server; + private final String ticket; + private final String appId; + private final String service; + private final String version; + private final String email; + private final boolean admin; + private final String authDomain; + private final Map attributes; + private final Timer wallTimer; + private final Long millisUntilSoftDeadline; + private final AppLogsWriter appLogsWriter; + + final Semaphore pendingApiCallSemaphore; + + final Semaphore runningApiCallSemaphore; + + /** + * Constructs a VM AppEngine API environment. + * + * @param server the host:port address of the VM's HTTP proxy server. + * @param ticket the request ticket (if null the default one will be computed). + * @param appId the application ID (required if ticket is null). + * @param service the module name (required if ticket is null). + * @param version the major application version (required if ticket is null). + * @param instance the VM instance ID (required if ticket is null). + * @param email the user's e-mail address (may be null). + * @param admin true if the user is an administrator. + * @param authDomain the user's authentication domain (may be null). + * @param wallTimer optional wall clock timer for the current request (required for deadline). + * @param millisUntilSoftDeadline optional soft deadline in milliseconds relative to 'wallTimer'. + * @param attributes map containing any attributes set on this environment. + */ + public ApiProxyEnvironment( + String server, String ticket, String appId, String service, + String version, String instance, String email, boolean admin, + String authDomain, Timer wallTimer, Long millisUntilSoftDeadline, + Map attributes) { + if (server == null || server.isEmpty()) { + throw new IllegalArgumentException("proxy server host:port must be specified"); + } + if (millisUntilSoftDeadline != null && wallTimer == null) { + throw new IllegalArgumentException("wallTimer required when setting millisUntilSoftDeadline"); + } + if (ticket == null || ticket.isEmpty()) { + if ((appId == null || appId.isEmpty()) || + (service == null || service.isEmpty()) || + (version == null || version.isEmpty()) || + (instance == null || instance.isEmpty())) { + throw new IllegalArgumentException( + "When ticket == null the following must be specified: appId=" + appId + + ", module=" + service + ", version=" + version + "instance=" + instance); + } + String escapedAppId = appId.replace(':', '_').replace('.', '_'); + this.ticket = escapedAppId + '/' + service + '.' + version + "." + instance; + } else { + this.ticket = ticket; + } + this.server = server; + this.appId = appId; + this.service = service == null ? "default" : service; + this.version = version; + this.email = email == null ? "" : email; + this.admin = admin; + this.authDomain = authDomain == null ? "" : authDomain; + this.wallTimer = wallTimer; + this.millisUntilSoftDeadline = millisUntilSoftDeadline; + this.attributes = Collections.synchronizedMap(attributes); + + this.appLogsWriter = new AppLogsWriter( + new LinkedList<>(), DEFAULT_FLUSH_APP_LOGS_EVERY_BYTE_COUNT, + DEFAULT_MAX_LOG_LINE_SIZE, MAX_LOG_FLUSH_SECONDS); + this.pendingApiCallSemaphore = new Semaphore(MAX_PENDING_API_CALLS); + this.runningApiCallSemaphore = new Semaphore(MAX_CONCURRENT_API_CALLS); + } + + public void addLogRecord(LogRecord record) { + appLogsWriter.addLogRecordAndMaybeFlush(record); + } + + public void flushLogs() { + appLogsWriter.flushAndWait(); + } + + public String getServer() { + return server; + } + + public String getTicket() { + return ticket; + } + + @Override + public String getAppId() { + return appId; + } + + @Override + public String getModuleId() { + return service; + } + + @Override + public String getVersionId() { + return version; + } + + @Override + public String getEmail() { + return email; + } + + @Override + public boolean isLoggedIn() { + return getEmail() != null && !getEmail().trim().isEmpty(); + } + + @Override + public boolean isAdmin() { + return admin; + } + + @Override + public String getAuthDomain() { + return authDomain; + } + + @Deprecated + @Override + public String getRequestNamespace() { + Object currentNamespace = + attributes.get(AttributeMapping.CURRENT_NAMESPACE_HEADER.attributeKey); + if (currentNamespace instanceof String) { + return (String) currentNamespace; + } + return ""; + } + + @Override + public Map getAttributes() { + return attributes; + } + + @Override + public long getRemainingMillis() { + if (millisUntilSoftDeadline == null) { + return Long.MAX_VALUE; + } + return millisUntilSoftDeadline - (wallTimer.getNanoseconds() / 1000000L); + } + + /** + * Notifies the environment that an API call was queued up. + * + * @throws ApiProxyException + */ + public void aSyncApiCallAdded(long maxWaitMs) throws ApiProxyException { + try { + if (pendingApiCallSemaphore.tryAcquire(maxWaitMs, TimeUnit.MILLISECONDS)) { + return; + } + throw new ApiProxyException("Timed out while acquiring a pending API call semaphore."); + } catch (InterruptedException e) { + throw new ApiProxyException( + "Thread interrupted while acquiring a pending API call semaphore."); + } + } + + /** + * Notifies the environment that an API call was started. + * + * @param releasePendingCall If true a pending call semaphore will be released (required if this + * API call was requested asynchronously). + * @throws ApiProxyException If the thread was interrupted while waiting for a semaphore. + */ + public void apiCallStarted(long maxWaitMs, boolean releasePendingCall) throws ApiProxyException { + try { + if (runningApiCallSemaphore.tryAcquire(maxWaitMs, TimeUnit.MILLISECONDS)) { + return; + } + throw new ApiProxyException("Timed out while acquiring an API call semaphore."); + } catch (InterruptedException e) { + throw new ApiProxyException("Thread interrupted while acquiring an API call semaphore."); + } finally { + if (releasePendingCall) { + pendingApiCallSemaphore.release(); + } + } + } + + /** + * Notifies the environment that an API call completed. + */ + public void apiCallCompleted() { + runningApiCallSemaphore.release(); + } +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java new file mode 100644 index 000000000..fa24342d3 --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java @@ -0,0 +1,294 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup; + +import com.google.appengine.repackaged.com.google.common.base.Stopwatch; +import com.google.appengine.repackaged.com.google.protobuf.ByteString; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.ApiConfig; +import com.google.apphosting.api.ApiProxy.LogRecord; +import com.google.apphosting.api.logservice.LogServicePb.FlushRequest; +import com.google.apphosting.api.logservice.LogServicePb.UserAppLogGroup; +import com.google.apphosting.api.logservice.LogServicePb.UserAppLogLine; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * {@code AppsLogWriter} is responsible for batching application logs + * for a single request and sending them back to the AppServer via the + * LogService.Flush API call. + *

+ *

The current algorithm used to send logs is as follows: + *

    + *
  • The code never allows more than {@code byteCountBeforeFlush} bytes of + * log data to accumulate in the buffer. If adding a new log line + * would exceed that limit, the current set of logs are removed from it and an + * asynchronous API call is started to flush the logs before buffering the + * new line.
    • + *

    + *

  • If another flush occurs while a previous flush is still + * pending, the caller will block synchronously until the previous + * call completed.
    • + *

    + *

  • When the overall request completes is should call @code{waitForCurrentFlushAndStartNewFlush} + * and report the flush count as a HTTP response header. The vm_runtime on the appserver + * will wait for the reported number of log flushes before forwarding the HTTP response + * to the user.
    • + *
+ *

+ *

This class is also responsible for splitting large log entries + * into smaller fragments, which is unrelated to the batching + * mechanism described above but is necessary to prevent the AppServer + * from truncating individual log entries. + *

+ *

This class is thread safe and all methods accessing local state are + * synchronized. Since each request have their own instance of this class the + * only contention possible is between the original request thread and and any + * child RequestThreads created by the request through the threading API. + */ +class AppLogsWriter { + private static final Logger logger = + Logger.getLogger(AppLogsWriter.class.getName()); + + static final String LOG_CONTINUATION_SUFFIX = "\n"; + static final int LOG_CONTINUATION_SUFFIX_LENGTH = LOG_CONTINUATION_SUFFIX.length(); + static final String LOG_CONTINUATION_PREFIX = "\n"; + static final int LOG_CONTINUATION_PREFIX_LENGTH = LOG_CONTINUATION_PREFIX.length(); + static final int MIN_MAX_LOG_MESSAGE_LENGTH = 1024; + static final int LOG_FLUSH_TIMEOUT_MS = 2000; + + private final int maxLogMessageLength; + private final int logCutLength; + private final int logCutLengthDiv10; + private final List buffer; + private final long maxBytesToFlush; + private long currentByteCount; + private final int maxSecondsBetweenFlush; + private int flushCount = 0; + private Future currentFlush; + private Stopwatch stopwatch; + + /** + * Construct an AppLogsWriter instance. + * + * @param buffer Buffer holding messages between flushes. + * @param maxBytesToFlush The maximum number of bytes of log message to + * allow in a single flush. The code flushes any cached logs before + * reaching this limit. If this is 0, AppLogsWriter will not start + * an intermediate flush based on size. + * @param maxLogMessageLength The maximum length of an individual log line. + * A single log line longer than this will be written as multiple log + * entries (with the continuation prefix/suffixes added to indicate this). + * @param maxFlushSeconds The amount of time to allow a log line to sit + * cached before flushing. Once a log line has been sitting for more + * than the specified time, all currently cached logs are flushed. If + * this is 0, no time based flushing occurs. + * N.B. because we only check the time on a log call, it is possible for + * a log to stay cached long after the specified time has been reached. + * Consider this example (assume maxFlushSeconds=60): the app logs a message + * when the handler starts but then does not log another message for 10 + * minutes. The initial log will stay cached until the second message + * is logged. + */ + public AppLogsWriter(List buffer, long maxBytesToFlush, int maxLogMessageLength, + int maxFlushSeconds) { + this.buffer = buffer; + this.maxSecondsBetweenFlush = maxFlushSeconds; + + if (maxLogMessageLength < MIN_MAX_LOG_MESSAGE_LENGTH) { + String message = String.format( + "maxLogMessageLength sillily small (%s); setting maxLogMessageLength to %s", + maxLogMessageLength, MIN_MAX_LOG_MESSAGE_LENGTH); + logger.warning(message); + this.maxLogMessageLength = MIN_MAX_LOG_MESSAGE_LENGTH; + } else { + this.maxLogMessageLength = maxLogMessageLength; + } + logCutLength = maxLogMessageLength - LOG_CONTINUATION_SUFFIX_LENGTH; + logCutLengthDiv10 = logCutLength / 10; + + if (maxBytesToFlush < this.maxLogMessageLength) { + String message = String.format( + "maxBytesToFlush (%s) smaller than maxLogMessageLength (%s)", + maxBytesToFlush, this.maxLogMessageLength); + logger.warning(message); + this.maxBytesToFlush = this.maxLogMessageLength; + } else { + this.maxBytesToFlush = maxBytesToFlush; + } + + stopwatch = Stopwatch.createUnstarted(); + } + + /** + * Add the specified {@link LogRecord} for the current request. If + * enough space (or in the future, time) has accumulated, an + * asynchronous flush may be started. If flushes are backed up, + * this method may block. + */ + synchronized void addLogRecordAndMaybeFlush(LogRecord fullRecord) { + for (LogRecord record : split(fullRecord)) { + UserAppLogLine logLine = UserAppLogLine.newBuilder() + .setLevel(record.getLevel().ordinal()) + .setTimestampUsec(record.getTimestamp()) + .setMessage(record.getMessage()) + .build(); + int maxEncodingSize = 1000; // logLine.maxEncodingSize(); + if (maxBytesToFlush > 0 && + (currentByteCount + maxEncodingSize) > maxBytesToFlush) { + logger.info(currentByteCount + " bytes of app logs pending, starting flush..."); + waitForCurrentFlushAndStartNewFlush(); + } + if (buffer.size() == 0) { + stopwatch.start(); + } + buffer.add(logLine); + currentByteCount += maxEncodingSize; + } + + if (maxSecondsBetweenFlush > 0 && + stopwatch.elapsed(TimeUnit.SECONDS) >= maxSecondsBetweenFlush) { + waitForCurrentFlushAndStartNewFlush(); + } + } + + /** + * Starts an asynchronous flush. This method may block if flushes + * are backed up. + * + * @return The number of times this AppLogsWriter has initiated a flush. + */ + synchronized int waitForCurrentFlushAndStartNewFlush() { + waitForCurrentFlush(); + if (buffer.size() > 0) { + currentFlush = doFlush(); + } + return flushCount; + } + + /** + * Initiates a synchronous flush. This method will always block + * until any pending flushes and its own flush completes. + */ + synchronized void flushAndWait() { + waitForCurrentFlush(); + if (buffer.size() > 0) { + currentFlush = doFlush(); + waitForCurrentFlush(); + } + } + + /** + * This method blocks until any outstanding flush is completed. This method + * should be called prior to {@link #doFlush()} so that it is impossible for + * the appserver to process logs out of order. + */ + private void waitForCurrentFlush() { + if (currentFlush != null) { + logger.info("Previous flush has not yet completed, blocking."); + try { + currentFlush.get( + ApiProxyDelegate.ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS + LOG_FLUSH_TIMEOUT_MS, + TimeUnit.MILLISECONDS); + } catch (InterruptedException ex) { + logger.warning("Interruped while blocking on a log flush, setting interrupt bit and " + + "continuing. Some logs may be lost or occur out of order!"); + Thread.currentThread().interrupt(); + } catch (TimeoutException e) { + logger.log(Level.WARNING, "Timeout waiting for log flush to complete. " + + "Log messages may have been lost/reordered!", e); + } catch (ExecutionException ex) { + logger.log( + Level.WARNING, + "A log flush request failed. Log messages may have been lost!", ex); + } + currentFlush = null; + } + } + + private Future doFlush() { + UserAppLogGroup.Builder group = UserAppLogGroup.newBuilder(); + for (UserAppLogLine logLine : buffer) { + group.addLogLine(logLine); + } + buffer.clear(); + currentByteCount = 0; + flushCount++; + stopwatch.reset(); + FlushRequest.Builder request = FlushRequest.newBuilder(); + request.setLogs(ByteString.copyFrom(group.build().toByteArray())); + ApiConfig apiConfig = new ApiConfig(); + apiConfig.setDeadlineInSeconds(LOG_FLUSH_TIMEOUT_MS / 1000.0); + return ApiProxy.makeAsyncCall("logservice", "Flush", request.build().toByteArray(), apiConfig); + } + + /** + * Because the App Server will truncate log messages that are too + * long, we want to split long log messages into mutliple messages. + * This method returns a {@link List} of {@code LogRecord}s, each of + * which have the same {@link LogRecord#getLevel()} and + * {@link LogRecord#getTimestamp()} as + * this one, and whose {@link LogRecord#getMessage()} is short enough + * that it will not be truncated by the App Server. If the + * {@code message} of this {@code LogRecord} is short enough, the list + * will contain only this {@code LogRecord}. Otherwise the list will + * contain multiple {@code LogRecord}s each of which contain a portion + * of the {@code message}. Additionally, strings will be + * prepended and appended to each of the {@code message}s indicating + * that the message is continued in the following log message or is a + * continuation of the previous log mesage. + */ + + List split(LogRecord aRecord) { + LinkedList theList = new LinkedList(); + String message = aRecord.getMessage(); + if (null == message || message.length() <= maxLogMessageLength) { + theList.add(aRecord); + return theList; + } + String remaining = message; + while (remaining.length() > 0) { + String nextMessage; + if (remaining.length() <= maxLogMessageLength) { + nextMessage = remaining; + remaining = ""; + } else { + int cutLength = logCutLength; + boolean cutAtNewline = false; + int friendlyCutLength = remaining.lastIndexOf('\n', logCutLength); + if (friendlyCutLength > logCutLengthDiv10) { + cutLength = friendlyCutLength; + cutAtNewline = true; + } + nextMessage = remaining.substring(0, cutLength) + LOG_CONTINUATION_SUFFIX; + remaining = remaining.substring(cutLength + (cutAtNewline ? 1 : 0)); + if (remaining.length() > maxLogMessageLength || + remaining.length() + LOG_CONTINUATION_PREFIX_LENGTH <= maxLogMessageLength) { + remaining = LOG_CONTINUATION_PREFIX + remaining; + } + } + theList.add(new LogRecord(aRecord, nextMessage)); + } + return theList; + } +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java new file mode 100644 index 000000000..a4bc8721b --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java @@ -0,0 +1,117 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.appengine.setup; + +import com.google.apphosting.api.ApiProxy; +import java.util.Map; +import java.util.function.Supplier; + +public class LazyApiProxyEnvironment implements ApiProxy.Environment { + private final Supplier supplier; + + private ApiProxyEnvironment delegate; + + public LazyApiProxyEnvironment(Supplier supplier) { + this.supplier = supplier; + } + + private ApiProxyEnvironment delegate() { + if (delegate == null) { + delegate = supplier.get(); + } + return delegate; + } + + @Override + public String getAppId() { + return delegate().getAppId(); + } + + @Override + public String getModuleId() { + return delegate().getModuleId(); + } + + @Override + public String getVersionId() { + return delegate().getVersionId(); + } + + @Override + public String getEmail() { + return delegate().getEmail(); + } + + @Override + public boolean isLoggedIn() { + return delegate().isLoggedIn(); + } + + @Override + public boolean isAdmin() { + return delegate().isAdmin(); + } + + @Override + public String getAuthDomain() { + return delegate().getAuthDomain(); + } + + @Override + @Deprecated + public String getRequestNamespace() { + return delegate().getRequestNamespace(); + } + + @Override + public Map getAttributes() { + return delegate().getAttributes(); + } + + @Override + public long getRemainingMillis() { + return delegate().getRemainingMillis(); + } + + public void aSyncApiCallAdded(long maxWaitMs) throws ApiProxy.ApiProxyException { + delegate().aSyncApiCallAdded(maxWaitMs); + } + + public void apiCallStarted(long maxWaitMs, boolean releasePendingCall) throws ApiProxy.ApiProxyException { + delegate().apiCallStarted(maxWaitMs, releasePendingCall); + + } + + public void apiCallCompleted() { + delegate().apiCallCompleted(); + } + + public void addLogRecord(ApiProxy.LogRecord record) { + delegate().addLogRecord(record); + } + + public void flushLogs() { + delegate().flushLogs(); + } + + public String getServer() { + return delegate().getServer(); // localhost:8089 + } + + public String getTicket() { + return delegate().getTicket(); + } +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java new file mode 100644 index 000000000..18d48615d --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup; + +import static com.google.appengine.repackaged.com.google.common.base.Preconditions.checkState; + +import com.google.appengine.repackaged.com.google.common.collect.ImmutableList; +import com.google.appengine.repackaged.com.google.common.collect.Lists; +import com.google.apphosting.api.ApiProxy; +import com.google.apphosting.api.ApiProxy.Environment; +import java.util.List; +import java.util.concurrent.ThreadFactory; + + +/** + * Thread factory creating threads with a request specific thread local environment. + */ +public class RequestThreadFactory implements ThreadFactory { + private final Environment requestEnvironment; + + private final Object mutex; + private final List createdThreads; + private volatile boolean allowNewRequestThreadCreation; + + /** + * Create a new VmRequestThreadFactory. + * + * @param requestEnvironment The request environment to install on each thread. + */ + public RequestThreadFactory(Environment requestEnvironment) { + this.mutex = new Object(); + this.requestEnvironment = requestEnvironment; + this.createdThreads = Lists.newLinkedList(); + this.allowNewRequestThreadCreation = true; + } + + /** + * Create a new {@link Thread} that executes {@code runnable} for the duration of the current + * request. This thread will be interrupted at the end of the current request. + * + * @param runnable The object whose run method is invoked when this thread is started. If null, + * this classes run method does nothing. + * @throws ApiProxy.ApiProxyException If called outside of a running request. + * @throws IllegalStateException If called after the request thread stops. + */ + @Override + public Thread newThread(final Runnable runnable) { + checkState(requestEnvironment != null, + "Request threads can only be created within the context of a running request."); + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + if (runnable == null) { + return; + } + checkState(allowNewRequestThreadCreation, + "Cannot start new threads after the request thread stops."); + ApiProxy.setEnvironmentForCurrentThread(requestEnvironment); + runnable.run(); + } + }); + checkState( + allowNewRequestThreadCreation, "Cannot create new threads after the request thread stops."); + synchronized (mutex) { + createdThreads.add(thread); + } + return thread; + } + + /** + * Returns an immutable copy of the current request thread list. + */ + public List getRequestThreads() { + synchronized (mutex) { + return ImmutableList.copyOf(createdThreads); + } + } +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java new file mode 100644 index 000000000..8674e2a25 --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup; + +import static com.google.appengine.repackaged.com.google.common.base.MoreObjects.firstNonNull; + +public class RuntimeUtils { + private static final String VM_API_PROXY_HOST = "appengine.googleapis.com"; + private static final int VM_API_PROXY_PORT = 10001; + public static final long ONE_DAY_IN_MILLIS = 24 * 60 * 60 * 1000; + public static final long MAX_USER_API_CALL_WAIT_MS = 60 * 1000; + + /** + * Returns the host:port of the API server. + * + * @return If environment variables API_HOST or API_PORT port are set the host and/or port is + * calculated from them. Otherwise the default host:port is used. + */ + public static String getApiServerAddress() { + String server = firstNonNull(System.getenv("API_HOST"), VM_API_PROXY_HOST); + String port = firstNonNull(System.getenv("API_PORT"), "" + VM_API_PROXY_PORT); + return server + ":" + port; + } +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java new file mode 100644 index 000000000..cba62dbbe --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup; + + +import com.google.appengine.setup.timer.AbstractIntervalTimer; + +/** + * Minimal implementation of com.google.apphosting.runtime.timer.Timer using only the system clock. + */ +public class TimerImpl extends AbstractIntervalTimer { + + @Override + protected long getCurrent() { + return System.nanoTime(); + } +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java new file mode 100644 index 000000000..caa8d50a5 --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java @@ -0,0 +1,82 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup.timer; + +/** + * {@code AbstractIntervalTimer} is common base class for {@link + * Timer} implementations that base measure the change in some value + * between the point where the timer is started and the point where + * the timer is stopped. + *

+ *

This class is thread-safe. + */ +abstract public class AbstractIntervalTimer implements Timer { + protected boolean running = false; + protected long startTime = 0L; + protected long cumulativeTime = 0L; + + public synchronized void start() { + if (running) { + throw new IllegalStateException("already running"); + } + + startTime = getCurrent(); + running = true; + } + + public synchronized void stop() { + if (!running) { + throw new IllegalStateException("not running"); + } + + update(getCurrent()); + running = false; + } + + public synchronized void update() { + update(getCurrent()); + } + + public long getNanoseconds() { + double ratio = getRatio(); + synchronized (this) { + if (running) { + return cumulativeTime + ((long) ((getCurrent() - startTime) * ratio)); + } else { + return cumulativeTime; + } + } + } + + /** + * The fraction of the change in the underlying counter which will + * be attributed to this timer. By default, 100% of it. + */ + protected double getRatio() { + return 1.0; + } + + protected void update(long currentValue) { + synchronized (this) { + long increment = (long) ((currentValue - startTime) * getRatio()); + cumulativeTime += increment; + startTime = currentValue; + } + } + + abstract protected long getCurrent(); +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java new file mode 100644 index 000000000..defb5aeca --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java @@ -0,0 +1,27 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup.timer; + +public interface Timer { + void start(); + + void stop(); + + long getNanoseconds(); + + void update(); +} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java new file mode 100644 index 000000000..e53dce573 --- /dev/null +++ b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java @@ -0,0 +1,22 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.appengine.setup.utils.http; + +@FunctionalInterface +public interface HttpRequest { + String getHeader(String name); +} From 21ec1a7ee18a4d87c7e347d26a90b447a043e22c Mon Sep 17 00:00:00 2001 From: Sriram Mahavadi Date: Fri, 4 Nov 2022 14:45:47 -0700 Subject: [PATCH 19/50] Internal change PiperOrigin-RevId: 486235380 Change-Id: I20df925cac63afb8fadf9d14b7dd713cf905cb6d --- .../appengine_setup/pom.xml | 46 -- .../appengine/setup/ApiProxyDelegate.java | 408 ----------------- .../appengine/setup/ApiProxyEnvironment.java | 429 ------------------ .../google/appengine/setup/AppLogsWriter.java | 294 ------------ .../setup/LazyApiProxyEnvironment.java | 117 ----- .../appengine/setup/RequestThreadFactory.java | 92 ---- .../google/appengine/setup/RuntimeUtils.java | 38 -- .../com/google/appengine/setup/TimerImpl.java | 31 -- .../setup/timer/AbstractIntervalTimer.java | 82 ---- .../google/appengine/setup/timer/Timer.java | 27 -- .../setup/utils/http/HttpRequest.java | 22 - 11 files changed, 1586 deletions(-) delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java delete mode 100644 google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml b/google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml deleted file mode 100644 index fe2db3481..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/pom.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - 4.0.0 - - com.google - appengine_setup - 1.0-SNAPSHOT - jar - AppEngine :: appengine_setup - DO NOT USE - Presently in Beta Mode. Library to help setup AppEngine features (bundled services, session management, etc). - - - 11 - 11 - true - - - - com.google.appengine - appengine-api-1.0-sdk - 2.0.8 - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - \ No newline at end of file diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java deleted file mode 100644 index 81c6d6eba..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyDelegate.java +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup; - -import com.google.appengine.repackaged.com.google.common.collect.Lists; -import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.api.ApiProxy.ApiConfig; -import com.google.apphosting.api.ApiProxy.ApiProxyException; -import com.google.apphosting.api.ApiProxy.LogRecord; -import com.google.apphosting.api.ApiProxy.RPCFailedException; -import com.google.apphosting.utils.remoteapi.RemoteApiPb; -import java.io.BufferedInputStream; -import java.io.IOException; -import java.util.List; -import java.util.Scanner; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.params.ConnManagerPNames; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.PoolingClientConnectionManager; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.CoreConnectionPNames; -import org.apache.http.params.HttpParams; -import org.apache.http.protocol.BasicHttpContext; - -/** - * Delegates AppEngine API calls to a local http API proxy. - *

- *

Instances should be registered using ApiProxy.setDelegate(ApiProxy.Delegate). - */ -public class ApiProxyDelegate implements ApiProxy.Delegate { - - private static final Logger logger = Logger.getLogger(ApiProxyDelegate.class.getName()); - - public static final String RPC_DEADLINE_HEADER = "X-Google-RPC-Service-Deadline"; - public static final String RPC_STUB_ID_HEADER = "X-Google-RPC-Service-Endpoint"; - public static final String RPC_METHOD_HEADER = "X-Google-RPC-Service-Method"; - - public static final String REQUEST_ENDPOINT = "/rpc_http"; // :8089 - public static final String REQUEST_STUB_ID = "app-engine-apis"; - public static final String REQUEST_STUB_METHOD = "/VMRemoteAPI.CallRemoteAPI"; - - protected static final String API_DEADLINE_KEY = - "com.google.apphosting.api.ApiProxy.api_deadline_key"; - - static final int ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS = 1000; - - protected int defaultTimeoutMs; - protected final ExecutorService executor; - - protected final HttpClient httpclient; - - final IdleConnectionMonitorThread monitorThread; - - private static ClientConnectionManager createConnectionManager() { - PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); - connectionManager.setMaxTotal(ApiProxyEnvironment.MAX_CONCURRENT_API_CALLS); - connectionManager.setDefaultMaxPerRoute(ApiProxyEnvironment.MAX_CONCURRENT_API_CALLS); - return connectionManager; - } - - public ApiProxyDelegate() { - this(new DefaultHttpClient(createConnectionManager())); - } - - - ApiProxyDelegate(HttpClient httpclient) { - this.defaultTimeoutMs = 5 * 60 * 1000; - this.executor = Executors.newCachedThreadPool(); - this.httpclient = httpclient; - this.monitorThread = new IdleConnectionMonitorThread(httpclient.getConnectionManager()); - this.monitorThread.start(); - } - - @Override - public byte[] makeSyncCall( - LazyApiProxyEnvironment environment, - String packageName, - String methodName, - byte[] requestData) - throws ApiProxyException { - return makeSyncCallWithTimeout(environment, packageName, methodName, requestData, - defaultTimeoutMs); - } - - private byte[] makeSyncCallWithTimeout( - LazyApiProxyEnvironment environment, - String packageName, - String methodName, - byte[] requestData, - int timeoutMs) - throws ApiProxyException { - return makeApiCall(environment, packageName, methodName, requestData, timeoutMs, false); - } - - private byte[] makeApiCall(LazyApiProxyEnvironment environment, - String packageName, - String methodName, - byte[] requestData, - int timeoutMs, - boolean wasAsync) { - environment.apiCallStarted(RuntimeUtils.MAX_USER_API_CALL_WAIT_MS, wasAsync); - try { - return runSyncCall(environment, packageName, methodName, requestData, timeoutMs); - } finally { - environment.apiCallCompleted(); - } - } - - - protected byte[] runSyncCall(LazyApiProxyEnvironment environment, String packageName, - String methodName, byte[] requestData, int timeoutMs) { - HttpPost request = createRequest(environment, packageName, methodName, requestData, timeoutMs); - try { - BasicHttpContext context = new BasicHttpContext(); - HttpResponse response = httpclient.execute(request, context); - - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - try (Scanner errorStreamScanner = - new Scanner(new BufferedInputStream(response.getEntity().getContent()));) { - logger.info("Error body: " + errorStreamScanner.useDelimiter("\\Z").next()); - throw new RPCFailedException(packageName, methodName); - } - } - try (BufferedInputStream bis = new BufferedInputStream(response.getEntity().getContent())) { - RemoteApiPb.Response remoteResponse = new RemoteApiPb.Response(); - if (!remoteResponse.parseFrom(bis)) { - logger.info( - "HTTP ApiProxy unable to parse response for " + packageName + "." + methodName); - throw new RPCFailedException(packageName, methodName); - } - if (remoteResponse.hasRpcError() || remoteResponse.hasApplicationError()) { - throw convertRemoteError(remoteResponse, packageName, methodName, logger); - } - return remoteResponse.getResponseAsBytes(); - } - } catch (IOException e) { - logger.info( - "HTTP ApiProxy I/O error for " + packageName + "." + methodName + ": " + e.getMessage()); - throw new RPCFailedException(packageName, methodName); - } finally { - request.releaseConnection(); - } - } - - /** - * Create an HTTP post request suitable for sending to the API server. - * - * @param environment The current VMApiProxyEnvironment - * @param packageName The API call package - * @param methodName The API call method - * @param requestData The POST payload. - * @param timeoutMs The timeout for this request - * @return an HttpPost object to send to the API. - */ - static HttpPost createRequest(LazyApiProxyEnvironment environment, String packageName, - String methodName, byte[] requestData, int timeoutMs) { - RemoteApiPb.Request remoteRequest = new RemoteApiPb.Request(); - remoteRequest.setServiceName(packageName); - remoteRequest.setMethod(methodName); - // Commenting below line to validate the use-cases where security ticket may be needed. So far we did not need. - //remoteRequest.setRequestId(environment.getTicket()); - remoteRequest.setRequestAsBytes(requestData); - - HttpPost request = new HttpPost("http://" + environment.getServer() + REQUEST_ENDPOINT); - request.setHeader(RPC_STUB_ID_HEADER, REQUEST_STUB_ID); - request.setHeader(RPC_METHOD_HEADER, REQUEST_STUB_METHOD); - - HttpParams params = new BasicHttpParams(); - params.setLongParameter(ConnManagerPNames.TIMEOUT, - timeoutMs + ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, - timeoutMs + ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS); - params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, - timeoutMs + ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS); - - params.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, Boolean.TRUE); - params.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, Boolean.FALSE); - request.setParams(params); - - Double deadline = (Double) (environment.getAttributes().get(API_DEADLINE_KEY)); - if (deadline == null) { - request.setHeader(RPC_DEADLINE_HEADER, - Double.toString(TimeUnit.SECONDS.convert(timeoutMs, TimeUnit.MILLISECONDS))); - } else { - request.setHeader(RPC_DEADLINE_HEADER, Double.toString(deadline)); - } - - Object dapperHeader = environment.getAttributes() - .get(ApiProxyEnvironment.AttributeMapping.DAPPER_ID.attributeKey); - if (dapperHeader instanceof String) { - request.setHeader( - ApiProxyEnvironment.AttributeMapping.DAPPER_ID.headerKey, (String) dapperHeader); - } - - ByteArrayEntity postPayload = new ByteArrayEntity(remoteRequest.toByteArray(), - ContentType.APPLICATION_OCTET_STREAM); - postPayload.setChunked(false); - request.setEntity(postPayload); - - return request; - } - - /** - * Convert RemoteApiPb.Response errors to the appropriate exception. - *

- *

The response must have exactly one of the RpcError and ApplicationError fields set. - * - * @param remoteResponse the Response - * @param packageName the name of the API package. - * @param methodName the name of the method within the API package. - * @param logger the Logger used to create log messages. - * @return ApiProxyException - */ - private static ApiProxyException convertRemoteError(RemoteApiPb.Response remoteResponse, - String packageName, String methodName, Logger logger) { - if (remoteResponse.hasRpcError()) { - return convertApiResponseRpcErrorToException( - remoteResponse.getRpcError(), - packageName, - methodName, - logger); - } - - RemoteApiPb.ApplicationError error = remoteResponse.getApplicationError(); - return new ApiProxy.ApplicationException(error.getCode(), error.getDetail()); - } - - /** - * Convert the RemoteApiPb.RpcError to the appropriate exception. - * - * @param rpcError the RemoteApiPb.RpcError. - * @param packageName the name of the API package. - * @param methodName the name of the method within the API package. - * @param logger the Logger used to create log messages. - * @return ApiProxyException - */ - private static ApiProxyException convertApiResponseRpcErrorToException( - RemoteApiPb.RpcError rpcError, String packageName, String methodName, Logger logger) { - - int rpcCode = rpcError.getCode(); - String errorDetail = rpcError.getDetail(); - if (rpcCode > RemoteApiPb.RpcError.ErrorCode.values().length) { - logger.severe("Received unrecognized error code from server: " + rpcError.getCode() + - " details: " + errorDetail); - return new ApiProxy.UnknownException(packageName, methodName); - } - RemoteApiPb.RpcError.ErrorCode errorCode = RemoteApiPb.RpcError.ErrorCode.values()[ - rpcError.getCode()]; - logger.warning("RPC failed : " + errorCode + " : " + errorDetail); - - switch (errorCode) { - case CALL_NOT_FOUND: - return new ApiProxy.CallNotFoundException(packageName, methodName); - case PARSE_ERROR: - return new ApiProxy.ArgumentException(packageName, methodName); - case SECURITY_VIOLATION: - logger.severe("Security violation: invalid request id used!"); - return new ApiProxy.UnknownException(packageName, methodName); - case CAPABILITY_DISABLED: - return new ApiProxy.CapabilityDisabledException( - errorDetail, packageName, methodName); - case OVER_QUOTA: - return new ApiProxy.OverQuotaException(packageName, methodName); - case REQUEST_TOO_LARGE: - return new ApiProxy.RequestTooLargeException(packageName, methodName); - case RESPONSE_TOO_LARGE: - return new ApiProxy.ResponseTooLargeException(packageName, methodName); - case BAD_REQUEST: - return new ApiProxy.ArgumentException(packageName, methodName); - case CANCELLED: - return new ApiProxy.CancelledException(packageName, methodName); - case FEATURE_DISABLED: - return new ApiProxy.FeatureNotEnabledException( - errorDetail, packageName, methodName); - case DEADLINE_EXCEEDED: - return new ApiProxy.ApiDeadlineExceededException(packageName, methodName); - default: - return new ApiProxy.UnknownException(packageName, methodName); - } - } - - private class MakeSyncCall implements Callable { - private final ApiProxyDelegate delegate; - private final LazyApiProxyEnvironment environment; - private final String packageName; - private final String methodName; - private final byte[] requestData; - private final int timeoutMs; - - public MakeSyncCall(ApiProxyDelegate delegate, - LazyApiProxyEnvironment environment, - String packageName, - String methodName, - byte[] requestData, - int timeoutMs) { - this.delegate = delegate; - this.environment = environment; - this.packageName = packageName; - this.methodName = methodName; - this.requestData = requestData; - this.timeoutMs = timeoutMs; - } - - @Override - public byte[] call() throws Exception { - return delegate.makeApiCall(environment, - packageName, - methodName, - requestData, - timeoutMs, - true); - } - } - - @Override - public Future makeAsyncCall( - LazyApiProxyEnvironment environment, - String packageName, - String methodName, - byte[] request, - ApiConfig apiConfig) { - int timeoutMs = defaultTimeoutMs; - if (apiConfig != null && apiConfig.getDeadlineInSeconds() != null) { - timeoutMs = (int) (apiConfig.getDeadlineInSeconds() * 1000); - } - environment.aSyncApiCallAdded(RuntimeUtils.MAX_USER_API_CALL_WAIT_MS); - return executor.submit(new MakeSyncCall(this, environment, packageName, - methodName, request, timeoutMs)); - } - - @Override - public void log(LazyApiProxyEnvironment environment, LogRecord record) { - if (environment != null) { - environment.addLogRecord(record); - } - } - - @Override - public void flushLogs(LazyApiProxyEnvironment environment) { - if (environment != null) { - environment.flushLogs(); - } - } - - @Override - public List getRequestThreads(LazyApiProxyEnvironment environment) { - Object threadFactory = - environment.getAttributes().get(ApiProxyEnvironment.REQUEST_THREAD_FACTORY_ATTR); - if (threadFactory != null && threadFactory instanceof RequestThreadFactory) { - return ((RequestThreadFactory) threadFactory).getRequestThreads(); - } - logger.warning("Got a call to getRequestThreads() but no VmRequestThreadFactory is available"); - return Lists.newLinkedList(); - } - - /** - * Simple connection watchdog verifying that our connections are alive. Any stale connections are - * cleared as well. - */ - class IdleConnectionMonitorThread extends Thread { - - private final ClientConnectionManager connectionManager; - - public IdleConnectionMonitorThread(ClientConnectionManager connectionManager) { - super("IdleApiConnectionMontorThread"); - this.connectionManager = connectionManager; - this.setDaemon(false); - } - - @Override - public void run() { - try { - while (true) { - connectionManager.closeExpiredConnections(); - connectionManager.closeIdleConnections(60, TimeUnit.SECONDS); - Thread.sleep(5000); - } - } catch (InterruptedException ex) { - } - } - } -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java deleted file mode 100644 index 789d95f15..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/ApiProxyEnvironment.java +++ /dev/null @@ -1,429 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup; - -import com.google.appengine.setup.timer.Timer; -import com.google.appengine.setup.utils.http.HttpRequest; -import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.api.ApiProxy.ApiProxyException; -import com.google.apphosting.api.ApiProxy.LogRecord; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; - -/** - * Implements the ApiProxy environment. - *

- * Supports instantiation within a request as well as outside the context of a request. - *

- * Instances should be registered using ApiProxy.setEnvironmentForCurrentThread(Environment). - */ -public class ApiProxyEnvironment implements ApiProxy.Environment { - - static final String GAE_APPLICATION = "GAE_APPLICATION"; - - static final String GAE_SERVICE = "GAE_SERVICE"; - - static final String GAE_VERSION = "GAE_VERSION"; - - static final String GAE_INSTANCE = "GAE_INSTANCE"; - - public static final String TICKET_HEADER = "X-AppEngine-Api-Ticket"; - public static final String EMAIL_HEADER = "X-AppEngine-User-Email"; - public static final String IS_ADMIN_HEADER = "X-AppEngine-User-Is-Admin"; - public static final String AUTH_DOMAIN_HEADER = "X-AppEngine-Auth-Domain"; - - public static final String BACKEND_ID_KEY = "com.google.appengine.backend.id"; - public static final String INSTANCE_ID_KEY = "com.google.appengine.instance.id"; - public static final String REQUEST_THREAD_FACTORY_ATTR = "com.google.appengine.api.ThreadManager.REQUEST_THREAD_FACTORY"; - public static final String BACKGROUND_THREAD_FACTORY_ATTR = "com.google.appengine.api.ThreadManager.BACKGROUND_THREAD_FACTORY"; - public static final String IS_FEDERATED_USER_KEY = "com.google.appengine.api.users.UserService.is_federated_user"; - public static final String IS_TRUSTED_IP_KEY = "com.google.appengine.runtime.is_trusted_ip"; - public static final String IS_TRUSTED_IP_HEADER = "X-AppEngine-Trusted-IP-Request"; - - private static final long DEFAULT_FLUSH_APP_LOGS_EVERY_BYTE_COUNT = 1024 * 1024L; - private static final int MAX_LOG_FLUSH_SECONDS = 60; - private static final int DEFAULT_MAX_LOG_LINE_SIZE = 8 * 1024; - - static final int MAX_CONCURRENT_API_CALLS = 100; - static final int MAX_PENDING_API_CALLS = 1000; - - /** - * Mapping from HTTP header keys to attribute keys. - */ - static enum AttributeMapping { - USER_ID( - "X-AppEngine-User-Id", - "com.google.appengine.api.users.UserService.user_id_key", - "", false), - USER_ORGANIZATION( - "X-AppEngine-User-Organization", - "com.google.appengine.api.users.UserService.user_organization", - "", false), - FEDERATED_IDENTITY( - "X-AppEngine-Federated-Identity", - "com.google.appengine.api.users.UserService.federated_identity", - "", false), - FEDERATED_PROVIDER( - "X-AppEngine-Federated-Provider", - "com.google.appengine.api.users.UserService.federated_authority", - "", false), - DATACENTER( - "X-AppEngine-Datacenter", - "com.google.apphosting.api.ApiProxy.datacenter", - "", false), - REQUEST_ID_HASH( - "X-AppEngine-Request-Id-Hash", - "com.google.apphosting.api.ApiProxy.request_id_hash", - null, false), - REQUEST_LOG_ID( - "X-AppEngine-Request-Log-Id", - "com.google.appengine.runtime.request_log_id", - null, false), - DAPPER_ID("X-Google-DapperTraceInfo", - "com.google.appengine.runtime.dapper_id", - null, false), - DEFAULT_VERSION_HOSTNAME( - "X-AppEngine-Default-Version-Hostname", - "com.google.appengine.runtime.default_version_hostname", - null, false), - DEFAULT_NAMESPACE_HEADER( - "X-AppEngine-Default-Namespace", - "com.google.appengine.api.NamespaceManager.appsNamespace", - null, false), - CURRENT_NAMESPACE_HEADER( - "X-AppEngine-Current-Namespace", - "com.google.appengine.api.NamespaceManager.currentNamespace", - null, false), - LOAS_PEER_USERNAME( - "X-AppEngine-LOAS-Peer-Username", - "com.google.net.base.peer.loas_peer_username", - "", true), - GAIA_ID( - "X-AppEngine-Gaia-Id", - "com.google.appengine.runtime.gaia_id", - "", true), - GAIA_AUTHUSER( - "X-AppEngine-Gaia-Authuser", - "com.google.appengine.runtime.gaia_authuser", - "", true), - GAIA_SESSION( - "X-AppEngine-Gaia-Session", - "com.google.appengine.runtime.gaia_session", - "", true), - APPSERVER_DATACENTER( - "X-AppEngine-Appserver-Datacenter", - "com.google.appengine.runtime.appserver_datacenter", - "", true), - APPSERVER_TASK_BNS( - "X-AppEngine-Appserver-Task-Bns", - "com.google.appengine.runtime.appserver_task_bns", - "", true); - - String headerKey; - String attributeKey; - Object defaultValue; - private boolean trustedAppOnly; - - /** - * Creates a mapping between an incoming request header and the thread local request attribute - * corresponding to that header. - * - * @param headerKey The HTTP header key. - * @param attributeKey The attribute key. - * @param defaultValue The default value to set if the header is missing, or null if no - * attribute should be set when the header is missing. - * @param trustedAppOnly If true the attribute should only be set for trusted apps. - */ - private AttributeMapping( - String headerKey, String attributeKey, Object defaultValue, boolean trustedAppOnly) { - this.headerKey = headerKey; - this.attributeKey = attributeKey; - this.defaultValue = defaultValue; - this.trustedAppOnly = trustedAppOnly; - } - } - - /** - * Helper method to use during the transition from metadata to environment variables. - * - * @param environmentMap the - * @param envKey The name of the environment variable to check first. - * @return If set the environment variable corresponding to envKey, the metadata entry otherwise. - */ - private static String getEnvOrMetadata(Map environmentMap, - String envKey) { - return environmentMap.get(envKey); - } - - public static ApiProxyEnvironment createFromHeaders(Map envMap, - HttpRequest request, - String server, - Timer wallTimer, - Long millisUntilSoftDeadline) { - String appId = getEnvOrMetadata(envMap, GAE_APPLICATION); - String module = getEnvOrMetadata(envMap, GAE_SERVICE); - String majorVersion = getEnvOrMetadata(envMap, GAE_VERSION); - String instance = getEnvOrMetadata(envMap, GAE_INSTANCE); - String ticket = request.getHeader(TICKET_HEADER); - String email = request.getHeader(EMAIL_HEADER); - boolean admin = false; - String value = request.getHeader(IS_ADMIN_HEADER); - if (value != null && !value.trim().isEmpty()) { - try { - admin = Integer.parseInt(value.trim()) != 0; - } catch (NumberFormatException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } - } - String authDomain = request.getHeader(AUTH_DOMAIN_HEADER); - boolean trustedApp = request.getHeader(IS_TRUSTED_IP_HEADER) != null; - - Map attributes = new HashMap<>(); - for (AttributeMapping mapping : AttributeMapping.values()) { - if (mapping.trustedAppOnly && !trustedApp) { - continue; - } - String headerValue = request.getHeader(mapping.headerKey); - if (headerValue != null) { - attributes.put(mapping.attributeKey, headerValue); - } else if (mapping.defaultValue != null) { - attributes.put(mapping.attributeKey, mapping.defaultValue); - } - } - - boolean federatedId = request.getHeader(AttributeMapping.FEDERATED_IDENTITY.headerKey) != null; - attributes.put(IS_FEDERATED_USER_KEY, federatedId); - - attributes.put(BACKEND_ID_KEY, module); - attributes.put(INSTANCE_ID_KEY, instance); - - if (trustedApp) { - boolean trustedIp = "1".equals(request.getHeader(IS_TRUSTED_IP_HEADER)); - attributes.put(IS_TRUSTED_IP_KEY, trustedIp); - } - - ApiProxyEnvironment requestEnvironment = new ApiProxyEnvironment(server, ticket, appId, - module, majorVersion, instance, email, admin, authDomain, - wallTimer, millisUntilSoftDeadline, attributes); - attributes.put(REQUEST_THREAD_FACTORY_ATTR, new RequestThreadFactory(requestEnvironment)); - attributes.put(BACKGROUND_THREAD_FACTORY_ATTR, Executors.defaultThreadFactory()); - - return requestEnvironment; - } - - private final String server; - private final String ticket; - private final String appId; - private final String service; - private final String version; - private final String email; - private final boolean admin; - private final String authDomain; - private final Map attributes; - private final Timer wallTimer; - private final Long millisUntilSoftDeadline; - private final AppLogsWriter appLogsWriter; - - final Semaphore pendingApiCallSemaphore; - - final Semaphore runningApiCallSemaphore; - - /** - * Constructs a VM AppEngine API environment. - * - * @param server the host:port address of the VM's HTTP proxy server. - * @param ticket the request ticket (if null the default one will be computed). - * @param appId the application ID (required if ticket is null). - * @param service the module name (required if ticket is null). - * @param version the major application version (required if ticket is null). - * @param instance the VM instance ID (required if ticket is null). - * @param email the user's e-mail address (may be null). - * @param admin true if the user is an administrator. - * @param authDomain the user's authentication domain (may be null). - * @param wallTimer optional wall clock timer for the current request (required for deadline). - * @param millisUntilSoftDeadline optional soft deadline in milliseconds relative to 'wallTimer'. - * @param attributes map containing any attributes set on this environment. - */ - public ApiProxyEnvironment( - String server, String ticket, String appId, String service, - String version, String instance, String email, boolean admin, - String authDomain, Timer wallTimer, Long millisUntilSoftDeadline, - Map attributes) { - if (server == null || server.isEmpty()) { - throw new IllegalArgumentException("proxy server host:port must be specified"); - } - if (millisUntilSoftDeadline != null && wallTimer == null) { - throw new IllegalArgumentException("wallTimer required when setting millisUntilSoftDeadline"); - } - if (ticket == null || ticket.isEmpty()) { - if ((appId == null || appId.isEmpty()) || - (service == null || service.isEmpty()) || - (version == null || version.isEmpty()) || - (instance == null || instance.isEmpty())) { - throw new IllegalArgumentException( - "When ticket == null the following must be specified: appId=" + appId + - ", module=" + service + ", version=" + version + "instance=" + instance); - } - String escapedAppId = appId.replace(':', '_').replace('.', '_'); - this.ticket = escapedAppId + '/' + service + '.' + version + "." + instance; - } else { - this.ticket = ticket; - } - this.server = server; - this.appId = appId; - this.service = service == null ? "default" : service; - this.version = version; - this.email = email == null ? "" : email; - this.admin = admin; - this.authDomain = authDomain == null ? "" : authDomain; - this.wallTimer = wallTimer; - this.millisUntilSoftDeadline = millisUntilSoftDeadline; - this.attributes = Collections.synchronizedMap(attributes); - - this.appLogsWriter = new AppLogsWriter( - new LinkedList<>(), DEFAULT_FLUSH_APP_LOGS_EVERY_BYTE_COUNT, - DEFAULT_MAX_LOG_LINE_SIZE, MAX_LOG_FLUSH_SECONDS); - this.pendingApiCallSemaphore = new Semaphore(MAX_PENDING_API_CALLS); - this.runningApiCallSemaphore = new Semaphore(MAX_CONCURRENT_API_CALLS); - } - - public void addLogRecord(LogRecord record) { - appLogsWriter.addLogRecordAndMaybeFlush(record); - } - - public void flushLogs() { - appLogsWriter.flushAndWait(); - } - - public String getServer() { - return server; - } - - public String getTicket() { - return ticket; - } - - @Override - public String getAppId() { - return appId; - } - - @Override - public String getModuleId() { - return service; - } - - @Override - public String getVersionId() { - return version; - } - - @Override - public String getEmail() { - return email; - } - - @Override - public boolean isLoggedIn() { - return getEmail() != null && !getEmail().trim().isEmpty(); - } - - @Override - public boolean isAdmin() { - return admin; - } - - @Override - public String getAuthDomain() { - return authDomain; - } - - @Deprecated - @Override - public String getRequestNamespace() { - Object currentNamespace = - attributes.get(AttributeMapping.CURRENT_NAMESPACE_HEADER.attributeKey); - if (currentNamespace instanceof String) { - return (String) currentNamespace; - } - return ""; - } - - @Override - public Map getAttributes() { - return attributes; - } - - @Override - public long getRemainingMillis() { - if (millisUntilSoftDeadline == null) { - return Long.MAX_VALUE; - } - return millisUntilSoftDeadline - (wallTimer.getNanoseconds() / 1000000L); - } - - /** - * Notifies the environment that an API call was queued up. - * - * @throws ApiProxyException - */ - public void aSyncApiCallAdded(long maxWaitMs) throws ApiProxyException { - try { - if (pendingApiCallSemaphore.tryAcquire(maxWaitMs, TimeUnit.MILLISECONDS)) { - return; - } - throw new ApiProxyException("Timed out while acquiring a pending API call semaphore."); - } catch (InterruptedException e) { - throw new ApiProxyException( - "Thread interrupted while acquiring a pending API call semaphore."); - } - } - - /** - * Notifies the environment that an API call was started. - * - * @param releasePendingCall If true a pending call semaphore will be released (required if this - * API call was requested asynchronously). - * @throws ApiProxyException If the thread was interrupted while waiting for a semaphore. - */ - public void apiCallStarted(long maxWaitMs, boolean releasePendingCall) throws ApiProxyException { - try { - if (runningApiCallSemaphore.tryAcquire(maxWaitMs, TimeUnit.MILLISECONDS)) { - return; - } - throw new ApiProxyException("Timed out while acquiring an API call semaphore."); - } catch (InterruptedException e) { - throw new ApiProxyException("Thread interrupted while acquiring an API call semaphore."); - } finally { - if (releasePendingCall) { - pendingApiCallSemaphore.release(); - } - } - } - - /** - * Notifies the environment that an API call completed. - */ - public void apiCallCompleted() { - runningApiCallSemaphore.release(); - } -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java deleted file mode 100644 index fa24342d3..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/AppLogsWriter.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup; - -import com.google.appengine.repackaged.com.google.common.base.Stopwatch; -import com.google.appengine.repackaged.com.google.protobuf.ByteString; -import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.api.ApiProxy.ApiConfig; -import com.google.apphosting.api.ApiProxy.LogRecord; -import com.google.apphosting.api.logservice.LogServicePb.FlushRequest; -import com.google.apphosting.api.logservice.LogServicePb.UserAppLogGroup; -import com.google.apphosting.api.logservice.LogServicePb.UserAppLogLine; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * {@code AppsLogWriter} is responsible for batching application logs - * for a single request and sending them back to the AppServer via the - * LogService.Flush API call. - *

- *

The current algorithm used to send logs is as follows: - *

    - *
  • The code never allows more than {@code byteCountBeforeFlush} bytes of - * log data to accumulate in the buffer. If adding a new log line - * would exceed that limit, the current set of logs are removed from it and an - * asynchronous API call is started to flush the logs before buffering the - * new line.
    • - *

    - *

  • If another flush occurs while a previous flush is still - * pending, the caller will block synchronously until the previous - * call completed.
    • - *

    - *

  • When the overall request completes is should call @code{waitForCurrentFlushAndStartNewFlush} - * and report the flush count as a HTTP response header. The vm_runtime on the appserver - * will wait for the reported number of log flushes before forwarding the HTTP response - * to the user.
    • - *
- *

- *

This class is also responsible for splitting large log entries - * into smaller fragments, which is unrelated to the batching - * mechanism described above but is necessary to prevent the AppServer - * from truncating individual log entries. - *

- *

This class is thread safe and all methods accessing local state are - * synchronized. Since each request have their own instance of this class the - * only contention possible is between the original request thread and and any - * child RequestThreads created by the request through the threading API. - */ -class AppLogsWriter { - private static final Logger logger = - Logger.getLogger(AppLogsWriter.class.getName()); - - static final String LOG_CONTINUATION_SUFFIX = "\n"; - static final int LOG_CONTINUATION_SUFFIX_LENGTH = LOG_CONTINUATION_SUFFIX.length(); - static final String LOG_CONTINUATION_PREFIX = "\n"; - static final int LOG_CONTINUATION_PREFIX_LENGTH = LOG_CONTINUATION_PREFIX.length(); - static final int MIN_MAX_LOG_MESSAGE_LENGTH = 1024; - static final int LOG_FLUSH_TIMEOUT_MS = 2000; - - private final int maxLogMessageLength; - private final int logCutLength; - private final int logCutLengthDiv10; - private final List buffer; - private final long maxBytesToFlush; - private long currentByteCount; - private final int maxSecondsBetweenFlush; - private int flushCount = 0; - private Future currentFlush; - private Stopwatch stopwatch; - - /** - * Construct an AppLogsWriter instance. - * - * @param buffer Buffer holding messages between flushes. - * @param maxBytesToFlush The maximum number of bytes of log message to - * allow in a single flush. The code flushes any cached logs before - * reaching this limit. If this is 0, AppLogsWriter will not start - * an intermediate flush based on size. - * @param maxLogMessageLength The maximum length of an individual log line. - * A single log line longer than this will be written as multiple log - * entries (with the continuation prefix/suffixes added to indicate this). - * @param maxFlushSeconds The amount of time to allow a log line to sit - * cached before flushing. Once a log line has been sitting for more - * than the specified time, all currently cached logs are flushed. If - * this is 0, no time based flushing occurs. - * N.B. because we only check the time on a log call, it is possible for - * a log to stay cached long after the specified time has been reached. - * Consider this example (assume maxFlushSeconds=60): the app logs a message - * when the handler starts but then does not log another message for 10 - * minutes. The initial log will stay cached until the second message - * is logged. - */ - public AppLogsWriter(List buffer, long maxBytesToFlush, int maxLogMessageLength, - int maxFlushSeconds) { - this.buffer = buffer; - this.maxSecondsBetweenFlush = maxFlushSeconds; - - if (maxLogMessageLength < MIN_MAX_LOG_MESSAGE_LENGTH) { - String message = String.format( - "maxLogMessageLength sillily small (%s); setting maxLogMessageLength to %s", - maxLogMessageLength, MIN_MAX_LOG_MESSAGE_LENGTH); - logger.warning(message); - this.maxLogMessageLength = MIN_MAX_LOG_MESSAGE_LENGTH; - } else { - this.maxLogMessageLength = maxLogMessageLength; - } - logCutLength = maxLogMessageLength - LOG_CONTINUATION_SUFFIX_LENGTH; - logCutLengthDiv10 = logCutLength / 10; - - if (maxBytesToFlush < this.maxLogMessageLength) { - String message = String.format( - "maxBytesToFlush (%s) smaller than maxLogMessageLength (%s)", - maxBytesToFlush, this.maxLogMessageLength); - logger.warning(message); - this.maxBytesToFlush = this.maxLogMessageLength; - } else { - this.maxBytesToFlush = maxBytesToFlush; - } - - stopwatch = Stopwatch.createUnstarted(); - } - - /** - * Add the specified {@link LogRecord} for the current request. If - * enough space (or in the future, time) has accumulated, an - * asynchronous flush may be started. If flushes are backed up, - * this method may block. - */ - synchronized void addLogRecordAndMaybeFlush(LogRecord fullRecord) { - for (LogRecord record : split(fullRecord)) { - UserAppLogLine logLine = UserAppLogLine.newBuilder() - .setLevel(record.getLevel().ordinal()) - .setTimestampUsec(record.getTimestamp()) - .setMessage(record.getMessage()) - .build(); - int maxEncodingSize = 1000; // logLine.maxEncodingSize(); - if (maxBytesToFlush > 0 && - (currentByteCount + maxEncodingSize) > maxBytesToFlush) { - logger.info(currentByteCount + " bytes of app logs pending, starting flush..."); - waitForCurrentFlushAndStartNewFlush(); - } - if (buffer.size() == 0) { - stopwatch.start(); - } - buffer.add(logLine); - currentByteCount += maxEncodingSize; - } - - if (maxSecondsBetweenFlush > 0 && - stopwatch.elapsed(TimeUnit.SECONDS) >= maxSecondsBetweenFlush) { - waitForCurrentFlushAndStartNewFlush(); - } - } - - /** - * Starts an asynchronous flush. This method may block if flushes - * are backed up. - * - * @return The number of times this AppLogsWriter has initiated a flush. - */ - synchronized int waitForCurrentFlushAndStartNewFlush() { - waitForCurrentFlush(); - if (buffer.size() > 0) { - currentFlush = doFlush(); - } - return flushCount; - } - - /** - * Initiates a synchronous flush. This method will always block - * until any pending flushes and its own flush completes. - */ - synchronized void flushAndWait() { - waitForCurrentFlush(); - if (buffer.size() > 0) { - currentFlush = doFlush(); - waitForCurrentFlush(); - } - } - - /** - * This method blocks until any outstanding flush is completed. This method - * should be called prior to {@link #doFlush()} so that it is impossible for - * the appserver to process logs out of order. - */ - private void waitForCurrentFlush() { - if (currentFlush != null) { - logger.info("Previous flush has not yet completed, blocking."); - try { - currentFlush.get( - ApiProxyDelegate.ADDITIONAL_HTTP_TIMEOUT_BUFFER_MS + LOG_FLUSH_TIMEOUT_MS, - TimeUnit.MILLISECONDS); - } catch (InterruptedException ex) { - logger.warning("Interruped while blocking on a log flush, setting interrupt bit and " + - "continuing. Some logs may be lost or occur out of order!"); - Thread.currentThread().interrupt(); - } catch (TimeoutException e) { - logger.log(Level.WARNING, "Timeout waiting for log flush to complete. " - + "Log messages may have been lost/reordered!", e); - } catch (ExecutionException ex) { - logger.log( - Level.WARNING, - "A log flush request failed. Log messages may have been lost!", ex); - } - currentFlush = null; - } - } - - private Future doFlush() { - UserAppLogGroup.Builder group = UserAppLogGroup.newBuilder(); - for (UserAppLogLine logLine : buffer) { - group.addLogLine(logLine); - } - buffer.clear(); - currentByteCount = 0; - flushCount++; - stopwatch.reset(); - FlushRequest.Builder request = FlushRequest.newBuilder(); - request.setLogs(ByteString.copyFrom(group.build().toByteArray())); - ApiConfig apiConfig = new ApiConfig(); - apiConfig.setDeadlineInSeconds(LOG_FLUSH_TIMEOUT_MS / 1000.0); - return ApiProxy.makeAsyncCall("logservice", "Flush", request.build().toByteArray(), apiConfig); - } - - /** - * Because the App Server will truncate log messages that are too - * long, we want to split long log messages into mutliple messages. - * This method returns a {@link List} of {@code LogRecord}s, each of - * which have the same {@link LogRecord#getLevel()} and - * {@link LogRecord#getTimestamp()} as - * this one, and whose {@link LogRecord#getMessage()} is short enough - * that it will not be truncated by the App Server. If the - * {@code message} of this {@code LogRecord} is short enough, the list - * will contain only this {@code LogRecord}. Otherwise the list will - * contain multiple {@code LogRecord}s each of which contain a portion - * of the {@code message}. Additionally, strings will be - * prepended and appended to each of the {@code message}s indicating - * that the message is continued in the following log message or is a - * continuation of the previous log mesage. - */ - - List split(LogRecord aRecord) { - LinkedList theList = new LinkedList(); - String message = aRecord.getMessage(); - if (null == message || message.length() <= maxLogMessageLength) { - theList.add(aRecord); - return theList; - } - String remaining = message; - while (remaining.length() > 0) { - String nextMessage; - if (remaining.length() <= maxLogMessageLength) { - nextMessage = remaining; - remaining = ""; - } else { - int cutLength = logCutLength; - boolean cutAtNewline = false; - int friendlyCutLength = remaining.lastIndexOf('\n', logCutLength); - if (friendlyCutLength > logCutLengthDiv10) { - cutLength = friendlyCutLength; - cutAtNewline = true; - } - nextMessage = remaining.substring(0, cutLength) + LOG_CONTINUATION_SUFFIX; - remaining = remaining.substring(cutLength + (cutAtNewline ? 1 : 0)); - if (remaining.length() > maxLogMessageLength || - remaining.length() + LOG_CONTINUATION_PREFIX_LENGTH <= maxLogMessageLength) { - remaining = LOG_CONTINUATION_PREFIX + remaining; - } - } - theList.add(new LogRecord(aRecord, nextMessage)); - } - return theList; - } -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java deleted file mode 100644 index a4bc8721b..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/LazyApiProxyEnvironment.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.appengine.setup; - -import com.google.apphosting.api.ApiProxy; -import java.util.Map; -import java.util.function.Supplier; - -public class LazyApiProxyEnvironment implements ApiProxy.Environment { - private final Supplier supplier; - - private ApiProxyEnvironment delegate; - - public LazyApiProxyEnvironment(Supplier supplier) { - this.supplier = supplier; - } - - private ApiProxyEnvironment delegate() { - if (delegate == null) { - delegate = supplier.get(); - } - return delegate; - } - - @Override - public String getAppId() { - return delegate().getAppId(); - } - - @Override - public String getModuleId() { - return delegate().getModuleId(); - } - - @Override - public String getVersionId() { - return delegate().getVersionId(); - } - - @Override - public String getEmail() { - return delegate().getEmail(); - } - - @Override - public boolean isLoggedIn() { - return delegate().isLoggedIn(); - } - - @Override - public boolean isAdmin() { - return delegate().isAdmin(); - } - - @Override - public String getAuthDomain() { - return delegate().getAuthDomain(); - } - - @Override - @Deprecated - public String getRequestNamespace() { - return delegate().getRequestNamespace(); - } - - @Override - public Map getAttributes() { - return delegate().getAttributes(); - } - - @Override - public long getRemainingMillis() { - return delegate().getRemainingMillis(); - } - - public void aSyncApiCallAdded(long maxWaitMs) throws ApiProxy.ApiProxyException { - delegate().aSyncApiCallAdded(maxWaitMs); - } - - public void apiCallStarted(long maxWaitMs, boolean releasePendingCall) throws ApiProxy.ApiProxyException { - delegate().apiCallStarted(maxWaitMs, releasePendingCall); - - } - - public void apiCallCompleted() { - delegate().apiCallCompleted(); - } - - public void addLogRecord(ApiProxy.LogRecord record) { - delegate().addLogRecord(record); - } - - public void flushLogs() { - delegate().flushLogs(); - } - - public String getServer() { - return delegate().getServer(); // localhost:8089 - } - - public String getTicket() { - return delegate().getTicket(); - } -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java deleted file mode 100644 index 18d48615d..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RequestThreadFactory.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup; - -import static com.google.appengine.repackaged.com.google.common.base.Preconditions.checkState; - -import com.google.appengine.repackaged.com.google.common.collect.ImmutableList; -import com.google.appengine.repackaged.com.google.common.collect.Lists; -import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.api.ApiProxy.Environment; -import java.util.List; -import java.util.concurrent.ThreadFactory; - - -/** - * Thread factory creating threads with a request specific thread local environment. - */ -public class RequestThreadFactory implements ThreadFactory { - private final Environment requestEnvironment; - - private final Object mutex; - private final List createdThreads; - private volatile boolean allowNewRequestThreadCreation; - - /** - * Create a new VmRequestThreadFactory. - * - * @param requestEnvironment The request environment to install on each thread. - */ - public RequestThreadFactory(Environment requestEnvironment) { - this.mutex = new Object(); - this.requestEnvironment = requestEnvironment; - this.createdThreads = Lists.newLinkedList(); - this.allowNewRequestThreadCreation = true; - } - - /** - * Create a new {@link Thread} that executes {@code runnable} for the duration of the current - * request. This thread will be interrupted at the end of the current request. - * - * @param runnable The object whose run method is invoked when this thread is started. If null, - * this classes run method does nothing. - * @throws ApiProxy.ApiProxyException If called outside of a running request. - * @throws IllegalStateException If called after the request thread stops. - */ - @Override - public Thread newThread(final Runnable runnable) { - checkState(requestEnvironment != null, - "Request threads can only be created within the context of a running request."); - Thread thread = new Thread(new Runnable() { - @Override - public void run() { - if (runnable == null) { - return; - } - checkState(allowNewRequestThreadCreation, - "Cannot start new threads after the request thread stops."); - ApiProxy.setEnvironmentForCurrentThread(requestEnvironment); - runnable.run(); - } - }); - checkState( - allowNewRequestThreadCreation, "Cannot create new threads after the request thread stops."); - synchronized (mutex) { - createdThreads.add(thread); - } - return thread; - } - - /** - * Returns an immutable copy of the current request thread list. - */ - public List getRequestThreads() { - synchronized (mutex) { - return ImmutableList.copyOf(createdThreads); - } - } -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java deleted file mode 100644 index 8674e2a25..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/RuntimeUtils.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup; - -import static com.google.appengine.repackaged.com.google.common.base.MoreObjects.firstNonNull; - -public class RuntimeUtils { - private static final String VM_API_PROXY_HOST = "appengine.googleapis.com"; - private static final int VM_API_PROXY_PORT = 10001; - public static final long ONE_DAY_IN_MILLIS = 24 * 60 * 60 * 1000; - public static final long MAX_USER_API_CALL_WAIT_MS = 60 * 1000; - - /** - * Returns the host:port of the API server. - * - * @return If environment variables API_HOST or API_PORT port are set the host and/or port is - * calculated from them. Otherwise the default host:port is used. - */ - public static String getApiServerAddress() { - String server = firstNonNull(System.getenv("API_HOST"), VM_API_PROXY_HOST); - String port = firstNonNull(System.getenv("API_PORT"), "" + VM_API_PROXY_PORT); - return server + ":" + port; - } -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java deleted file mode 100644 index cba62dbbe..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/TimerImpl.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup; - - -import com.google.appengine.setup.timer.AbstractIntervalTimer; - -/** - * Minimal implementation of com.google.apphosting.runtime.timer.Timer using only the system clock. - */ -public class TimerImpl extends AbstractIntervalTimer { - - @Override - protected long getCurrent() { - return System.nanoTime(); - } -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java deleted file mode 100644 index caa8d50a5..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/AbstractIntervalTimer.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup.timer; - -/** - * {@code AbstractIntervalTimer} is common base class for {@link - * Timer} implementations that base measure the change in some value - * between the point where the timer is started and the point where - * the timer is stopped. - *

- *

This class is thread-safe. - */ -abstract public class AbstractIntervalTimer implements Timer { - protected boolean running = false; - protected long startTime = 0L; - protected long cumulativeTime = 0L; - - public synchronized void start() { - if (running) { - throw new IllegalStateException("already running"); - } - - startTime = getCurrent(); - running = true; - } - - public synchronized void stop() { - if (!running) { - throw new IllegalStateException("not running"); - } - - update(getCurrent()); - running = false; - } - - public synchronized void update() { - update(getCurrent()); - } - - public long getNanoseconds() { - double ratio = getRatio(); - synchronized (this) { - if (running) { - return cumulativeTime + ((long) ((getCurrent() - startTime) * ratio)); - } else { - return cumulativeTime; - } - } - } - - /** - * The fraction of the change in the underlying counter which will - * be attributed to this timer. By default, 100% of it. - */ - protected double getRatio() { - return 1.0; - } - - protected void update(long currentValue) { - synchronized (this) { - long increment = (long) ((currentValue - startTime) * getRatio()); - cumulativeTime += increment; - startTime = currentValue; - } - } - - abstract protected long getCurrent(); -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java deleted file mode 100644 index defb5aeca..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/timer/Timer.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup.timer; - -public interface Timer { - void start(); - - void stop(); - - long getNanoseconds(); - - void update(); -} diff --git a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java b/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java deleted file mode 100644 index e53dce573..000000000 --- a/google3/third_party/java_src/appengine_standard/appengine_setup/src/main/java/com/google/appengine/setup/utils/http/HttpRequest.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.setup.utils.http; - -@FunctionalInterface -public interface HttpRequest { - String getHeader(String name); -} From a4dfb9da2cc264058137a17aefc545f5e24364fe Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Fri, 4 Nov 2022 15:34:17 -0700 Subject: [PATCH 20/50] Upgrade our last internal dependant jar proto1.jar containing proto buffers compiled with internal Google proto1 compiler and JDK8. PiperOrigin-RevId: 486245629 Change-Id: Iad405b4ba12530d729179b9dc5aaeceb3d169311 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 83244db9e..8e9455c04 100644 --- a/pom.xml +++ b/pom.xml @@ -381,7 +381,7 @@ com.google.appengine proto1 - 0.6 + 0.7 com.google.cloud.datastore From ae0a751d46f2575464b277d5edc1ae8364f8310f Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Fri, 4 Nov 2022 16:00:54 -0700 Subject: [PATCH 21/50] Internal change PiperOrigin-RevId: 486250945 Change-Id: I1433badffa26673cd6dc54ab9a3446249b827304 --- kokoro/gcp_ubuntu/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kokoro/gcp_ubuntu/release.sh b/kokoro/gcp_ubuntu/release.sh index 69d451cd2..c0cbd588f 100644 --- a/kokoro/gcp_ubuntu/release.sh +++ b/kokoro/gcp_ubuntu/release.sh @@ -98,8 +98,8 @@ git checkout -b $RELEASE_NUMBER # Make sure `JAVA_HOME` is set. echo "JAVA_HOME = $JAVA_HOME" # compile all packages -echo "Calling release:prepare and perform." -./mvnw release:prepare release:perform -B -q --settings=../settings.xml -DskipTests -Darguments=-DskipTests -Dgpg.homedir=${GNUPGHOME} -Dgpg.passphrase=${GPG_PASSPHRASE} +echo "Calling release:prepare and perform in verbose mode." +./mvnw release:prepare release:perform -B -X --settings=../settings.xml -DskipTests -Darguments=-DskipTests -Dgpg.homedir=${GNUPGHOME} -Dgpg.passphrase=${GPG_PASSPHRASE} git config user.email gae-java-bot@google.com git config user.name gae-java-bot From 7fc6161bcc50e7fd1895243a4475b9e057124f24 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Fri, 4 Nov 2022 17:09:32 -0700 Subject: [PATCH 22/50] Create version 2.0.11-SNAPSHOT after pushing 2.0.10 to Maven central. PiperOrigin-RevId: 486263635 Change-Id: I7bb0858ebd2c12d879135cbd938d0415c40782fa --- TRYLATESTBITSINPROD.md | 4 ++-- api/pom.xml | 2 +- api_dev/pom.xml | 2 +- api_legacy/pom.xml | 2 +- appengine-api-1.0-sdk/pom.xml | 2 +- appengine-api-stubs/pom.xml | 2 +- appengine_jsr107/pom.xml | 2 +- appengine_resources/pom.xml | 2 +- appengine_testing/pom.xml | 2 +- appengine_testing_tests/pom.xml | 2 +- applications/pom.xml | 2 +- applications/proberapp/pom.xml | 2 +- .../appengine_standard/api_compatibility_tests/pom.xml | 2 +- .../google/appengine/apicompat/NoSerializeImmutableTest.java | 2 +- .../appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java | 2 +- kokoro/gcp_ubuntu/release.sh | 4 ++-- lib/pom.xml | 2 +- lib/tools_api/pom.xml | 2 +- lib/xml_validator/pom.xml | 2 +- lib/xml_validator_test/pom.xml | 2 +- local_runtime_shared/pom.xml | 2 +- pom.xml | 2 +- protobuf/pom.xml | 2 +- quickstartgenerator/pom.xml | 2 +- remoteapi/pom.xml | 2 +- runtime/deployment/pom.xml | 2 +- runtime/impl/pom.xml | 2 +- runtime/local/pom.xml | 2 +- runtime/main/pom.xml | 2 +- runtime/pom.xml | 2 +- runtime/test/pom.xml | 2 +- runtime/testapps/pom.xml | 2 +- runtime/util/pom.xml | 2 +- runtime_shared/pom.xml | 2 +- sdk_assembly/pom.xml | 2 +- sessiondata/pom.xml | 2 +- shared_sdk/pom.xml | 2 +- third_party/geronimo_javamail/pom.xml | 2 +- utils/pom.xml | 2 +- 39 files changed, 41 insertions(+), 41 deletions(-) diff --git a/TRYLATESTBITSINPROD.md b/TRYLATESTBITSINPROD.md index 9bfd2b755..f9ab8fa23 100644 --- a/TRYLATESTBITSINPROD.md +++ b/TRYLATESTBITSINPROD.md @@ -43,12 +43,12 @@ top of your web application and change the entrypoint to boot with these jars in mvn clean install ``` -Let's assume the current built version is `2.0.10-SNAPSHOT`. +Let's assume the current built version is `2.0.11-SNAPSHOT`. Add the dependency for the GAE runtime jars in your application pom.xml file: ``` - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT ${appengine.runtime.location} ... diff --git a/api/pom.xml b/api/pom.xml index 442e203a3..f11d07057 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT true diff --git a/api_dev/pom.xml b/api_dev/pom.xml index 89320cc32..be00f0dcb 100644 --- a/api_dev/pom.xml +++ b/api_dev/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/api_legacy/pom.xml b/api_legacy/pom.xml index 402a4d218..1ccec9074 100644 --- a/api_legacy/pom.xml +++ b/api_legacy/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/appengine-api-1.0-sdk/pom.xml b/appengine-api-1.0-sdk/pom.xml index 62dcfd901..e8ee57418 100644 --- a/appengine-api-1.0-sdk/pom.xml +++ b/appengine-api-1.0-sdk/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar AppEngine :: appengine-api-1.0-sdk diff --git a/appengine-api-stubs/pom.xml b/appengine-api-stubs/pom.xml index 06bf62328..6c2346a4d 100644 --- a/appengine-api-stubs/pom.xml +++ b/appengine-api-stubs/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/appengine_jsr107/pom.xml b/appengine_jsr107/pom.xml index 34782ef78..6725c1afc 100644 --- a/appengine_jsr107/pom.xml +++ b/appengine_jsr107/pom.xml @@ -24,7 +24,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT diff --git a/appengine_resources/pom.xml b/appengine_resources/pom.xml index 05e7d8cb0..fb41fbb7a 100644 --- a/appengine_resources/pom.xml +++ b/appengine_resources/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar AppEngine :: appengine-resources diff --git a/appengine_testing/pom.xml b/appengine_testing/pom.xml index f52d6e558..5511b50b9 100644 --- a/appengine_testing/pom.xml +++ b/appengine_testing/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/appengine_testing_tests/pom.xml b/appengine_testing_tests/pom.xml index f70823427..e816d2679 100644 --- a/appengine_testing_tests/pom.xml +++ b/appengine_testing_tests/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/applications/pom.xml b/applications/pom.xml index 0c2e60ceb..07439b4b6 100644 --- a/applications/pom.xml +++ b/applications/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT pom diff --git a/applications/proberapp/pom.xml b/applications/proberapp/pom.xml index 2ed2ed881..8339a4ca0 100644 --- a/applications/proberapp/pom.xml +++ b/applications/proberapp/pom.xml @@ -27,7 +27,7 @@ com.google.appengine applications - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml index 37b419898..f75a84285 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java index 83b8a7d41..c3bd60e48 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/NoSerializeImmutableTest.java @@ -84,7 +84,7 @@ public class NoSerializeImmutableTest { */ @Test public void serializableCollectionFieldsAreNotGuavaImmutable() throws Exception { - File appengineApiJar = new File("/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.10-SNAPSHOT.jar"); + File appengineApiJar = new File("/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.11-SNAPSHOT.jar"); assertThat(appengineApiJar.exists()).isTrue(); ClassLoader apiJarClassLoader = new URLClassLoader(new URL[] {appengineApiJar.toURI().toURL()}); Class messageLite = diff --git a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java index 77a0ab55a..c9ad3ab22 100644 --- a/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java +++ b/google3/third_party/java_src/appengine_standard/api_compatibility_tests/src/test/java/com/google/appengine/apicompat/usage/ApiExhaustiveUsageTestCase.java @@ -58,7 +58,7 @@ public abstract class ApiExhaustiveUsageTestCase { * The path to the sdk api jar. */ private static final String API_JAR_PATH - = "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.10-SNAPSHOT.jar"; + = "/tmp/check_build/appengine-api-1.0-sdk/target/appengine-api-1.0-sdk-2.0.11-SNAPSHOT.jar"; private boolean isExhaustiveUsageClass(String clsName) { return clsName.startsWith("com.google.appengine.apicompat.usage"); diff --git a/kokoro/gcp_ubuntu/release.sh b/kokoro/gcp_ubuntu/release.sh index c0cbd588f..69d451cd2 100644 --- a/kokoro/gcp_ubuntu/release.sh +++ b/kokoro/gcp_ubuntu/release.sh @@ -98,8 +98,8 @@ git checkout -b $RELEASE_NUMBER # Make sure `JAVA_HOME` is set. echo "JAVA_HOME = $JAVA_HOME" # compile all packages -echo "Calling release:prepare and perform in verbose mode." -./mvnw release:prepare release:perform -B -X --settings=../settings.xml -DskipTests -Darguments=-DskipTests -Dgpg.homedir=${GNUPGHOME} -Dgpg.passphrase=${GPG_PASSPHRASE} +echo "Calling release:prepare and perform." +./mvnw release:prepare release:perform -B -q --settings=../settings.xml -DskipTests -Darguments=-DskipTests -Dgpg.homedir=${GNUPGHOME} -Dgpg.passphrase=${GPG_PASSPHRASE} git config user.email gae-java-bot@google.com git config user.name gae-java-bot diff --git a/lib/pom.xml b/lib/pom.xml index 2f3323690..44350e089 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT pom diff --git a/lib/tools_api/pom.xml b/lib/tools_api/pom.xml index f943b96af..3fdc0df63 100644 --- a/lib/tools_api/pom.xml +++ b/lib/tools_api/pom.xml @@ -23,7 +23,7 @@ com.google.appengine lib-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/lib/xml_validator/pom.xml b/lib/xml_validator/pom.xml index 740a4ef26..318c38e5e 100644 --- a/lib/xml_validator/pom.xml +++ b/lib/xml_validator/pom.xml @@ -22,7 +22,7 @@ com.google.appengine lib-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar AppEngine :: libxmlvalidator diff --git a/lib/xml_validator_test/pom.xml b/lib/xml_validator_test/pom.xml index 4e033cb8b..660a12302 100644 --- a/lib/xml_validator_test/pom.xml +++ b/lib/xml_validator_test/pom.xml @@ -22,7 +22,7 @@ com.google.appengine lib-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar AppEngine :: libxmlvalidator_test diff --git a/local_runtime_shared/pom.xml b/local_runtime_shared/pom.xml index 207517a2e..98d414ec9 100644 --- a/local_runtime_shared/pom.xml +++ b/local_runtime_shared/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar AppEngine :: appengine-local-runtime-shared diff --git a/pom.xml b/pom.xml index 8e9455c04..af7d92f8f 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 4.0.0 com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT pom AppEngine :: Parent project diff --git a/protobuf/pom.xml b/protobuf/pom.xml index 8978ee58d..689d0345e 100644 --- a/protobuf/pom.xml +++ b/protobuf/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/quickstartgenerator/pom.xml b/quickstartgenerator/pom.xml index 5591eda42..190b25a94 100644 --- a/quickstartgenerator/pom.xml +++ b/quickstartgenerator/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/remoteapi/pom.xml b/remoteapi/pom.xml index 7318d8f80..973b896c6 100644 --- a/remoteapi/pom.xml +++ b/remoteapi/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar AppEngine :: appengine-remote-api diff --git a/runtime/deployment/pom.xml b/runtime/deployment/pom.xml index 81c9108f7..0ea18dfe1 100644 --- a/runtime/deployment/pom.xml +++ b/runtime/deployment/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT pom diff --git a/runtime/impl/pom.xml b/runtime/impl/pom.xml index 14d4a9e52..8c3e7e36c 100644 --- a/runtime/impl/pom.xml +++ b/runtime/impl/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/runtime/local/pom.xml b/runtime/local/pom.xml index ad09fb0eb..d4bfbb2a3 100644 --- a/runtime/local/pom.xml +++ b/runtime/local/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/runtime/main/pom.xml b/runtime/main/pom.xml index b86504466..ad4696291 100644 --- a/runtime/main/pom.xml +++ b/runtime/main/pom.xml @@ -23,7 +23,7 @@ com.google.appengine runtime-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/runtime/pom.xml b/runtime/pom.xml index 91f968431..747301cbc 100644 --- a/runtime/pom.xml +++ b/runtime/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT AppEngine :: runtime projects pom diff --git a/runtime/test/pom.xml b/runtime/test/pom.xml index ba24e2d19..c4dc4991e 100644 --- a/runtime/test/pom.xml +++ b/runtime/test/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/runtime/testapps/pom.xml b/runtime/testapps/pom.xml index 81537a20c..526dc774d 100644 --- a/runtime/testapps/pom.xml +++ b/runtime/testapps/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/runtime/util/pom.xml b/runtime/util/pom.xml index ce121f3df..e7ac20cf6 100644 --- a/runtime/util/pom.xml +++ b/runtime/util/pom.xml @@ -22,7 +22,7 @@ com.google.appengine runtime-parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/runtime_shared/pom.xml b/runtime_shared/pom.xml index aecc4030d..e791c7e14 100644 --- a/runtime_shared/pom.xml +++ b/runtime_shared/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/sdk_assembly/pom.xml b/sdk_assembly/pom.xml index 1fda37130..4c14ef867 100644 --- a/sdk_assembly/pom.xml +++ b/sdk_assembly/pom.xml @@ -20,7 +20,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT 4.0.0 appengine-java-sdk diff --git a/sessiondata/pom.xml b/sessiondata/pom.xml index bbae9f127..c40130875 100644 --- a/sessiondata/pom.xml +++ b/sessiondata/pom.xml @@ -23,7 +23,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/shared_sdk/pom.xml b/shared_sdk/pom.xml index 8a590c158..808799de3 100644 --- a/shared_sdk/pom.xml +++ b/shared_sdk/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT jar diff --git a/third_party/geronimo_javamail/pom.xml b/third_party/geronimo_javamail/pom.xml index c5ef4662f..9bbce3ff1 100644 --- a/third_party/geronimo_javamail/pom.xml +++ b/third_party/geronimo_javamail/pom.xml @@ -22,7 +22,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT ../../pom.xml diff --git a/utils/pom.xml b/utils/pom.xml index af1ed9352..3cab69990 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -21,7 +21,7 @@ com.google.appengine parent - 2.0.10-SNAPSHOT + 2.0.11-SNAPSHOT true From 983f3dfbc2d8d71a7ba1e3e619c7d2f26e9ef25e Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Thu, 10 Nov 2022 17:07:43 -0800 Subject: [PATCH 23/50] Add source code download location for the activation Maven dependency in the SDK distribution. PiperOrigin-RevId: 487678370 Change-Id: I93a73c045cd799b2641d8986708e847ddd33d308 --- .../src/main/resources/README.activation | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 sdk_assembly/src/main/resources/README.activation diff --git a/sdk_assembly/src/main/resources/README.activation b/sdk_assembly/src/main/resources/README.activation new file mode 100644 index 000000000..82511643e --- /dev/null +++ b/sdk_assembly/src/main/resources/README.activation @@ -0,0 +1,18 @@ +# +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +The source code for the activation maven dependency can be downloaded from: +https://maven-central-asia.storage-download.googleapis.com/maven2/javax/activation/activation/1.1.1/activation-1.1.1-sources.jar From 516976ea3d2a71a7a02260ba2220361fc1c36326 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Fri, 11 Nov 2022 16:05:17 -0800 Subject: [PATCH 24/50] Internal change PiperOrigin-RevId: 487939097 Change-Id: I3a9995104f3ad2284bd78680e22aeca3dae2daed --- .../com/google/apphosting/runtime/JavaRuntimeFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java index 09a4448c4..c12496d1c 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java @@ -299,8 +299,8 @@ private static void maybeConfigureProfiler(JavaRuntimeParams params) { profilerConfigClass.getMethod("setHeapProfilerEnabled", boolean.class); setCpuProfilerEnabled.invoke(profilerConfig, params.getEnableCloudCpuProfiler()); setHeapProfilerEnabled.invoke(profilerConfig, params.getEnableCloudHeapProfiler()); - Method start = profilerClass.getMethod("start", profilerConfigClass); - start.invoke(null, profilerConfig); + // Method start = profilerClass.getMethod("start", profilerConfigClass); + // TODO (bs/258855140) start.invoke(null, profilerConfig); } catch (Exception e) { logger.atWarning().withCause(e).log("Failed to start the profiler"); } From 3c31edbb0e0ba60ac6fed29397a4ab867bde7021 Mon Sep 17 00:00:00 2001 From: GAE Java Team Date: Sun, 13 Nov 2022 23:04:59 -0800 Subject: [PATCH 25/50] Internal change PiperOrigin-RevId: 488272950 Change-Id: I51cbd1b71f53f1ff15460a6c722f51b107bf6e7a --- .../runtime/jetty94/JettyHttpProxy.java | 7 +- .../jetty94/JettyServletEngineAdapter.java | 5 +- .../runtime/jetty94/SizeLimitHandler.java | 201 ++++++++++++++++++ 3 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/SizeLimitHandler.java diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java index 27051f1db..182437df2 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyHttpProxy.java @@ -71,6 +71,7 @@ public class JettyHttpProxy { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); private static final String JETTY_LOG_CLASS = "org.eclipse.jetty.util.log.class"; private static final String JETTY_STDERRLOG = "org.eclipse.jetty.util.log.StdErrLog"; + private static final long MAX_REQUEST_SIZE = 32 * 1024 * 1024; /** * Based on the adapter configuration, this will start a new Jetty server in charge of proxying @@ -110,11 +111,15 @@ public static Server newServer( config.setSendServerVersion(false); config.setSendXPoweredBy(false); + SizeLimitHandler sizeLimitHandler = new SizeLimitHandler(MAX_REQUEST_SIZE, -1); + sizeLimitHandler.setHandler(handler); + GzipHandler gzip = new GzipHandler(); gzip.setIncludedMethods("GET", "POST"); gzip.setInflateBufferSize(8 * 1024); + gzip.setHandler(sizeLimitHandler); + server.setHandler(gzip); - gzip.setHandler(handler); logger.atInfo().log("Starting Jetty http server for Java runtime proxy."); return server; diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServletEngineAdapter.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServletEngineAdapter.java index 0eede951e..e6acfff96 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServletEngineAdapter.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/JettyServletEngineAdapter.java @@ -49,6 +49,7 @@ public class JettyServletEngineAdapter implements ServletEngineAdapter { private static final String DEFAULT_APP_YAML_PATH = "/WEB-INF/appengine-generated/app.yaml"; private static final int MIN_THREAD_POOL_THREADS = 0; private static final int MAX_THREAD_POOL_THREADS = 100; + private static final long MAX_RESPONSE_SIZE = 32 * 1024 * 1024; private AppVersionKey lastAppVersionKey; static { @@ -100,7 +101,9 @@ public void start(String serverInfo, ServletEngineAdapter.Config runtimeOptions) new AppVersionHandlerFactory( server, serverInfo, contextFactory, /*useJettyErrorPageHandler=*/ false); appVersionHandlerMap = new AppVersionHandlerMap(appVersionHandlerFactory); - server.setHandler(appVersionHandlerMap); + SizeLimitHandler sizeLimitHandler = new SizeLimitHandler(-1, MAX_RESPONSE_SIZE); + sizeLimitHandler.setHandler(appVersionHandlerMap); + server.setHandler(sizeLimitHandler); if (runtimeOptions.useJettyHttpProxy()) { server.setAttribute( diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/SizeLimitHandler.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/SizeLimitHandler.java new file mode 100644 index 000000000..3757e8551 --- /dev/null +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/jetty94/SizeLimitHandler.java @@ -0,0 +1,201 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.apphosting.runtime.jetty94; + +import java.io.IOException; +import java.nio.ByteBuffer; +import javax.annotation.Nullable; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; +import org.eclipse.jetty.http.BadMessageException; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.server.HttpInput; +import org.eclipse.jetty.server.HttpOutput; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.HandlerWrapper; +import org.eclipse.jetty.util.Callback; + +/** + * A handler that can limit the size of message bodies in requests and responses. + * + *

The optional request and response limits are imposed by checking the {@code Content-Length} + * header or observing the actual bytes seen by the handler. Handler order is important, in as much + * as if this handler is before a the {@link org.eclipse.jetty.server.handler.gzip.GzipHandler}, + * then it will limit compressed sized, if it as after the {@link + * org.eclipse.jetty.server.handler.gzip.GzipHandler} then the limit is applied to uncompressed + * bytes. If a size limit is exceeded then {@link BadMessageException} is thrown with a {@link + * org.eclipse.jetty.http.HttpStatus#PAYLOAD_TOO_LARGE_413} status. + */ +public class SizeLimitHandler extends HandlerWrapper { + private final long requestLimit; + private final long responseLimit; + + /** + * @param requestLimit The request body size limit in bytes or -1 for no limit + * @param responseLimit The response body size limit in bytes or -1 for no limit + */ + public SizeLimitHandler(long requestLimit, long responseLimit) { + this.requestLimit = requestLimit; + this.responseLimit = responseLimit; + } + + protected void checkRequestLimit(long size) { + if (requestLimit >= 0 && size > requestLimit) { + throw new BadMessageException(413, "Request body is too large: " + size + ">" + requestLimit); + } + } + + protected void checkResponseLimit(long size) { + if (responseLimit >= 0 && size > responseLimit) { + throw new BadMessageException( + 500, "Response body is too large: " + size + ">" + responseLimit); + } + } + + @Override + public void handle( + String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException { + if (requestLimit >= 0 || responseLimit >= 0) { + HttpOutput httpOutput = baseRequest.getResponse().getHttpOutput(); + HttpOutput.Interceptor interceptor = httpOutput.getInterceptor(); + LimitInterceptor limit = new LimitInterceptor(interceptor); + + if (requestLimit >= 0) { + long contentLength = baseRequest.getContentLengthLong(); + checkRequestLimit(contentLength); + if (contentLength < 0) { + baseRequest.getHttpInput().addInterceptor(limit); + } + } + + if (responseLimit > 0) { + httpOutput.setInterceptor(limit); + response = new LimitResponse(response); + } + } + + super.handle(target, baseRequest, request, response); + } + + private class LimitInterceptor implements HttpOutput.Interceptor, HttpInput.Interceptor { + private final HttpOutput.Interceptor nextOutput; + long read; + long written; + + public LimitInterceptor(HttpOutput.Interceptor nextOutput) { + this.nextOutput = nextOutput; + } + + @Override + public HttpOutput.Interceptor getNextInterceptor() { + return nextOutput; + } + + @Override + public boolean isOptimizedForDirectBuffers() { + return nextOutput.isOptimizedForDirectBuffers(); + } + + @Nullable + @Override + public HttpInput.Content readFrom(HttpInput.Content content) { + if (content == null) { + return null; + } + + if (content.hasContent()) { + read += content.remaining(); + checkResponseLimit(read); + } + return content; + } + + @Override + public void write(ByteBuffer content, boolean last, Callback callback) { + if (content.hasRemaining()) { + written += content.remaining(); + + try { + checkResponseLimit(written); + } catch (Throwable t) { + callback.failed(t); + return; + } + } + getNextInterceptor().write(content, last, callback); + } + + @Override + public void resetBuffer() { + written = 0; + getNextInterceptor().resetBuffer(); + } + } + + private class LimitResponse extends HttpServletResponseWrapper { + public LimitResponse(HttpServletResponse response) { + super(response); + } + + @Override + public void setContentLength(int len) { + checkResponseLimit(len); + super.setContentLength(len); + } + + @Override + public void setContentLengthLong(long len) { + checkResponseLimit(len); + super.setContentLengthLong(len); + } + + @Override + public void setHeader(String name, String value) { + if (HttpHeader.CONTENT_LENGTH.is(name)) { + checkResponseLimit(Long.parseLong(value)); + } + super.setHeader(name, value); + } + + @Override + public void addHeader(String name, String value) { + if (HttpHeader.CONTENT_LENGTH.is(name)) { + checkResponseLimit(Long.parseLong(value)); + } + super.addHeader(name, value); + } + + @Override + public void setIntHeader(String name, int value) { + if (HttpHeader.CONTENT_LENGTH.is(name)) { + checkResponseLimit(value); + } + super.setIntHeader(name, value); + } + + @Override + public void addIntHeader(String name, int value) { + if (HttpHeader.CONTENT_LENGTH.is(name)) { + checkResponseLimit(value); + } + super.addIntHeader(name, value); + } + } +} From bfd46a547bcb2b14298a639c19c47990836534c6 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Tue, 22 Nov 2022 13:05:51 -0800 Subject: [PATCH 26/50] Internal change PiperOrigin-RevId: 490316551 Change-Id: I12e65004f0d845ce9ad9c5e3e76b7c090cc4f513 --- .../com/google/apphosting/runtime/JavaRuntimeFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java index c12496d1c..09a4448c4 100644 --- a/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java +++ b/runtime/impl/src/main/java/com/google/apphosting/runtime/JavaRuntimeFactory.java @@ -299,8 +299,8 @@ private static void maybeConfigureProfiler(JavaRuntimeParams params) { profilerConfigClass.getMethod("setHeapProfilerEnabled", boolean.class); setCpuProfilerEnabled.invoke(profilerConfig, params.getEnableCloudCpuProfiler()); setHeapProfilerEnabled.invoke(profilerConfig, params.getEnableCloudHeapProfiler()); - // Method start = profilerClass.getMethod("start", profilerConfigClass); - // TODO (bs/258855140) start.invoke(null, profilerConfig); + Method start = profilerClass.getMethod("start", profilerConfigClass); + start.invoke(null, profilerConfig); } catch (Exception e) { logger.atWarning().withCause(e).log("Failed to start the profiler"); } From 59271c43224b79112bac30522957af747ec797e3 Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Mon, 28 Nov 2022 11:33:50 -0800 Subject: [PATCH 27/50] Bump to newer artifacts version. PiperOrigin-RevId: 491399156 Change-Id: I6b4af0dc4dca5ab8539685da7b68f7021f410c32 --- pom.xml | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/pom.xml b/pom.xml index af7d92f8f..e2011c5e3 100644 --- a/pom.xml +++ b/pom.xml @@ -355,12 +355,12 @@ com.google.api-client google-api-client-appengine - 2.0.0 + 2.0.1 com.google.api-client google-api-client - 2.0.0 + 2.0.1 com.google.appengine @@ -386,7 +386,7 @@ com.google.cloud.datastore datastore-v1-proto-client - 2.11.5 + 2.12.5 com.google.geometry @@ -433,13 +433,13 @@ com.google.auto.value auto-value - 1.9 + 1.10 provided com.google.auto.value auto-value-annotations - 1.9 + 1.10 com.esotericsoftware.yamlbeans @@ -449,12 +449,12 @@ com.google.api.grpc proto-google-cloud-datastore-v1 - 0.103.0 + 0.103.5 com.google.api.grpc proto-google-common-protos - 2.9.6 + 2.10.0 com.google.code.findbugs @@ -464,7 +464,7 @@ com.google.code.gson gson - 2.9.1 + 2.10 com.google.flogger @@ -485,12 +485,12 @@ com.google.errorprone error_prone_annotations - 2.15.0 + 2.16 com.google.http-client google-http-client - 1.42.2 + 1.42.3 com.google.http-client @@ -506,12 +506,12 @@ com.google.protobuf protobuf-java - 3.21.7 + 3.21.9 com.google.protobuf protobuf-java-util - 3.21.7 + 3.21.9 javax.activation @@ -521,7 +521,7 @@ com.google.appengine geronimo-javamail_1.4_spec - 1.4.1 + 1.4.4-2.0.10 javax.servlet @@ -551,7 +551,7 @@ org.apache.maven.plugin-tools maven-plugin-annotations - 3.6.4 + 3.7.0 provided @@ -562,7 +562,7 @@ org.checkerframework checker-qual - 3.24.0 + 3.27.0 provided @@ -580,7 +580,7 @@ org.glassfish.web el-impl - 2.2 + 2.2.1-b05 quartz @@ -616,7 +616,7 @@ com.google.http-client google-http-client-appengine - 1.42.2 + 1.42.3 com.google.oauth-client @@ -626,22 +626,22 @@ io.grpc grpc-api - 1.49.2 + 1.50.2 io.grpc grpc-stub - 1.49.2 + 1.50.2 io.grpc grpc-protobuf - 1.49.2 + 1.50.2 io.grpc grpc-netty - 1.49.2 + 1.50.2 org.apache.tomcat @@ -651,12 +651,12 @@ com.fasterxml.jackson.core jackson-core - 2.13.4 + 2.14.0 joda-time joda-time - 2.11.2 + 2.12.1 org.json @@ -697,13 +697,13 @@ org.mockito mockito-core - 4.7.0 + 4.8.1 test org.mockito mockito-junit-jupiter - 4.8.0 + 4.8.1 test @@ -770,7 +770,7 @@ org.apache.maven.plugins maven-release-plugin - 3.0.0-M6 + 3.0.0-M7 false deploy @@ -850,7 +850,7 @@ org.codehaus.mojo versions-maven-plugin - 2.12.0 + 2.13.0 From 3e687b708555ac91fb42914a8bca48835f18d15c Mon Sep 17 00:00:00 2001 From: Ludovic Champenois Date: Mon, 28 Nov 2022 15:28:44 -0800 Subject: [PATCH 28/50] Cleanup: delete unused DevAppServer2 java code. PiperOrigin-RevId: 491459455 Change-Id: Iaa42da7dcfeb80207f9dce28a7e013a4db1d5a11 --- .../development/IsolatedAppClassLoader.java | 9 +- .../IsolatedAppClassLoaderTest.java | 67 -- .../DevAppServer2ClassLoader.java | 97 -- .../devappserver2/DevAppServer2Delegate.java | 122 --- .../devappserver2/DevAppServer2Factory.java | 80 -- .../devappserver2/DevAppServer2Impl.java | 425 -------- .../devappserver2/RemoteApiDelegate.java | 106 -- .../development/devappserver2/RemoteRpc.java | 167 ---- .../devappserver2/RequestIdFilter.java | 57 -- .../devappserver2/StandaloneInstance.java | 175 ---- .../webdefault/jetty9/webdefault.xml | 919 ------------------ runtime/local/pom.xml | 1 - 12 files changed, 1 insertion(+), 2224 deletions(-) delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2ClassLoader.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Delegate.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Factory.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Impl.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteApiDelegate.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteRpc.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RequestIdFilter.java delete mode 100644 lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/StandaloneInstance.java delete mode 100644 lib/tools_api/src/main/resources/com/google/appengine/tools/development/devappserver2/webdefault/jetty9/webdefault.xml diff --git a/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java b/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java index 237f73e05..4ab1b59bb 100644 --- a/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java +++ b/api_dev/src/main/java/com/google/appengine/tools/development/IsolatedAppClassLoader.java @@ -46,11 +46,9 @@ public class IsolatedAppClassLoader extends URLClassLoader { private static final Logger logger = Logger.getLogger(IsolatedAppClassLoader.class.getName()); - // Web-default.xml files for Jetty9 based devappserver1 and devappserver2. + // Web-default.xml files for Jetty9 based devappserver1. private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVER1 = "com/google/appengine/tools/development/jetty9/webdefault.xml"; - private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVER2 = - "com/google/appengine/tools/development/devappserver2/webdefault/jetty9/webdefault.xml"; // This task queue related servlet should be loaded by the application classloader when the // api jar is used by the application, and default to the runtime classloader when the application @@ -84,11 +82,6 @@ public IsolatedAppClassLoader(File appRoot, File externalResourceDir, URL[] urls IsolatedAppClassLoader.class .getClassLoader() .getResourceAsStream(WEB_DEFAULT_LOCATION_DEVAPPSERVER1))) - .addAll( - getServletAndFilterClasses( - IsolatedAppClassLoader.class - .getClassLoader() - .getResourceAsStream(WEB_DEFAULT_LOCATION_DEVAPPSERVER2))) .build(); } diff --git a/api_dev/src/test/java/com/google/appengine/tools/development/IsolatedAppClassLoaderTest.java b/api_dev/src/test/java/com/google/appengine/tools/development/IsolatedAppClassLoaderTest.java index ccdcfa896..ea6baaade 100644 --- a/api_dev/src/test/java/com/google/appengine/tools/development/IsolatedAppClassLoaderTest.java +++ b/api_dev/src/test/java/com/google/appengine/tools/development/IsolatedAppClassLoaderTest.java @@ -31,8 +31,6 @@ public final class IsolatedAppClassLoaderTest { private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVER1_PATH = "com/google/appengine/tools/development/jetty9/webdefault.xml"; - private static final String WEB_DEFAULT_LOCATION_DEVAPPSERVER2_PATH = - "com/google/appengine/tools/development/devappserver2/webdefault/jetty9/webdefault.xml"; @Test @org.junit.Ignore @@ -100,71 +98,6 @@ public void calculateCorrectContentForServletsFiltersDevappServer1() throws Exce assertThat(classes).containsExactlyElementsIn(classesFromWebDefault1); } - @Test - @org.junit.Ignore - public void calculateCorrectContentForServletsFiltersDevappServer2() throws Exception { - Set classesWebDefault2 = - getClassesInAppDefinition(WEB_DEFAULT_LOCATION_DEVAPPSERVER2_PATH); - assertThat(classesWebDefault2).hasSize(54); - Set classesFromWebDefault2 = - ImmutableSet.of( - "com.google.appengine.tools.development.jetty9.StaticFileFilter", - "com.google.apphosting.utils.servlet.TransactionCleanupFilter", - "com.google.appengine.tools.development.HeaderVerificationFilter", - "com.google.appengine.tools.development.jetty9.ResponseRewriterFilterJetty9", - "com.google.appengine.tools.development.devappserver2.RequestIdFilter", - "com.google.appengine.tools.development.jetty9.LocalResourceFileServlet", - "com.google.appengine.api.blobstore.dev.UploadBlobServlet", - "com.google.appengine.api.images.dev.LocalBlobImageServlet", - "com.google.appengine.tools.development.jetty9.FixupJspServlet", - "com.google.appengine.api.users.dev.LocalLoginServlet", - "com.google.appengine.api.users.dev.LocalLogoutServlet", - "com.google.appengine.api.users.dev.LocalOAuthRequestTokenServlet", - "com.google.appengine.api.users.dev.LocalOAuthAuthorizeTokenServlet", - "com.google.appengine.api.users.dev.LocalOAuthAccessTokenServlet", - "com.google.apphosting.utils.servlet.DeferredTaskServlet", - "com.google.apphosting.utils.servlet.SessionCleanupServlet", - "com.google.apphosting.utils.servlet.CapabilitiesStatusServlet", - "com.google.apphosting.utils.servlet.DatastoreViewerServlet", - "com.google.apphosting.utils.servlet.ModulesServlet", - "com.google.apphosting.utils.servlet.TaskQueueViewerServlet", - "com.google.apphosting.utils.servlet.InboundMailServlet", - "com.google.apphosting.utils.servlet.SearchServlet", - "com.google.apphosting.utils.servlet.AdminConsoleResourceServlet", - "org.apache.jsp.ah.jetty9.adminConsole_jsp", - "org.apache.jsp.ah.jetty9.datastoreViewerHead_jsp", - "org.apache.jsp.ah.jetty9.datastoreViewerBody_jsp", - "org.apache.jsp.ah.jetty9.datastoreViewerFinal_jsp", - "org.apache.jsp.ah.jetty9.searchIndexesListHead_jsp", - "org.apache.jsp.ah.jetty9.searchIndexesListBody_jsp", - "org.apache.jsp.ah.jetty9.searchIndexesListFinal_jsp", - "org.apache.jsp.ah.jetty9.searchIndexHead_jsp", - "org.apache.jsp.ah.jetty9.searchIndexBody_jsp", - "org.apache.jsp.ah.jetty9.searchIndexFinal_jsp", - "org.apache.jsp.ah.jetty9.searchDocumentHead_jsp", - "org.apache.jsp.ah.jetty9.searchDocumentBody_jsp", - "org.apache.jsp.ah.jetty9.searchDocumentFinal_jsp", - "org.apache.jsp.ah.jetty9.capabilitiesStatusHead_jsp", - "org.apache.jsp.ah.jetty9.capabilitiesStatusBody_jsp", - "org.apache.jsp.ah.jetty9.capabilitiesStatusFinal_jsp", - "org.apache.jsp.ah.jetty9.entityDetailsHead_jsp", - "org.apache.jsp.ah.jetty9.entityDetailsBody_jsp", - "org.apache.jsp.ah.jetty9.entityDetailsFinal_jsp", - "org.apache.jsp.ah.jetty9.indexDetailsHead_jsp", - "org.apache.jsp.ah.jetty9.indexDetailsBody_jsp", - "org.apache.jsp.ah.jetty9.indexDetailsFinal_jsp", - "org.apache.jsp.ah.jetty9.modulesHead_jsp", - "org.apache.jsp.ah.jetty9.modulesBody_jsp", - "org.apache.jsp.ah.jetty9.modulesFinal_jsp", - "org.apache.jsp.ah.jetty9.taskqueueViewerHead_jsp", - "org.apache.jsp.ah.jetty9.taskqueueViewerBody_jsp", - "org.apache.jsp.ah.jetty9.taskqueueViewerFinal_jsp", - "org.apache.jsp.ah.jetty9.inboundMailHead_jsp", - "org.apache.jsp.ah.jetty9.inboundMailBody_jsp", - "org.apache.jsp.ah.jetty9.inboundMailFinal_jsp"); - assertThat(classesWebDefault2).containsExactlyElementsIn(classesFromWebDefault2); - } - private static Set getClassesInAppDefinition(String appDefPath) throws Exception { URL resourceURL = Resources.getResource(appDefPath); try (InputStream stream = resourceURL.openStream()) { diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2ClassLoader.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2ClassLoader.java deleted file mode 100644 index 848f727d0..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2ClassLoader.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import com.google.appengine.tools.info.AppengineSdk; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.List; - -/** - * Isolates the DevAppServer and all of its dependencies into its own classloader. - * This ClassLoader refuses to load anything off of the JVM's System ClassLoader - * except for JRE classes (i.e. it ignores classpath and JAR manifest entries). - * - */ -class DevAppServer2ClassLoader extends URLClassLoader { - - private final ClassLoader delegate; - - private static final String DEV_APP_SERVER_INTERFACE - = "com.google.appengine.tools.development.DevAppServer"; - - private static final String APP_CONTEXT_INTERFACE - = "com.google.appengine.tools.development.AppContext"; - - private static final String DEV_SOCKET_IMPL_FACTORY - = "com.google.appengine.tools.development.DevSocketImplFactory"; - - /** - * Creates a new {@code DevAppServerClassLoader}, which will load - * libraries from the App Engine Sdk. - * - * @param delegate A delegate ClassLoader from which a few shared - * classes will be loaded (e.g. DevAppServer). - */ - static DevAppServer2ClassLoader newClassLoader(ClassLoader delegate) { - // NB Doing shared, then impl, in order, allows us to prefer - // returning shared classes when asked by other classloaders. This makes - // it so that we don't have to have the impl and shared classes - // be a strictly disjoint set. - List libs = new ArrayList(AppengineSdk.getSdk().getSharedLibs()); - libs.addAll(AppengineSdk.getSdk().getImplLibs()); - // Needed by admin console servlets, which are loaded by this - // ClassLoader - libs.addAll(AppengineSdk.getSdk().getUserJspLibs()); - return new DevAppServer2ClassLoader(libs.toArray(new URL[libs.size()]), delegate); - } - - // NB - // - // Isolating our code may seem seem like overkill, but it's really necessary - // in terms of integration scenarios, such as with GWT. In general, we've - // discovered that users tend to put all sorts of nasty things on the - // system classpath, and it interferes with our ability to properly isolate - // user code and assign correct permissions. - - DevAppServer2ClassLoader(URL[] urls, ClassLoader delegate) { - super(urls, null); - this.delegate = delegate; - } - - @Override - protected synchronized Class loadClass(String name, boolean resolve) - throws ClassNotFoundException { - - // Special-case a few classes that need to be shared. - if (name.equals(DEV_APP_SERVER_INTERFACE) - || name.equals(APP_CONTEXT_INTERFACE) - || name.equals(DEV_SOCKET_IMPL_FACTORY) - || name.startsWith("com.google.appengine.tools.info.")) { - Class c = delegate.loadClass(name); - if (resolve) { - resolveClass(c); - } - return c; - } - - // Otherwise, we're safe to load anything returned from the JRE or from - // ourselves. - return super.loadClass(name, resolve); - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Delegate.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Delegate.java deleted file mode 100644 index 68340f118..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Delegate.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import com.google.appengine.tools.development.DevLogService; -import com.google.appengine.tools.development.DevServices; -import com.google.appengine.tools.remoteapi.RemoteApiOptions; -import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.api.ApiProxy.Environment; -import java.util.List; -import java.util.concurrent.Future; -import java.util.logging.ConsoleHandler; -import java.util.logging.Formatter; -import java.util.logging.Handler; -import java.util.logging.LogRecord; -import java.util.logging.StreamHandler; - -/** - * ApiProxy.Delegate that handles API calls by making an RPC to a remote API server. Because of the - * structure of the Java Remote API client support, we must construct a separate delegate for every - * thread that might make RPCs. This class hides that by creating per-thread delegates on demand and - * forwarding every API call to the current thread's delegate. It also avoids infinite recursion by - * ensuring that the HTTP request wrapping the RPC uses native sockets rather than being diverted to - * the URLFetch service. - * - */ -class DevAppServer2Delegate implements ApiProxy.Delegate, DevServices { - private final ThreadLocal threadLocalRemoteApiDelegate = new ThreadLocal<>(); - private final RemoteApiOptions remoteApiOptions; - - DevAppServer2Delegate(RemoteApiOptions remoteApiOptions) { - this.remoteApiOptions = remoteApiOptions; - } - - /** - * Return a remote API delegate for the current thread. This may be a new object or the cached - * result of an earlier call to this method from the same thread. - */ - private synchronized ApiProxy.Delegate remoteApiDelegate() { - RemoteApiDelegate delegate = threadLocalRemoteApiDelegate.get(); - if (delegate == null) { - RemoteRpc remoteRpc = new RemoteRpc(remoteApiOptions); - delegate = new RemoteApiDelegate(remoteRpc, remoteApiOptions); - threadLocalRemoteApiDelegate.set(delegate); - } - return delegate; - } - - @Override - public byte[] makeSyncCall( - Environment environment, String packageName, String methodName, byte[] request) { - return remoteApiDelegate().makeSyncCall(environment, packageName, methodName, request); - } - - @Override - public Future makeAsyncCall( - Environment environment, - String packageName, - String methodName, - byte[] request, - ApiProxy.ApiConfig apiConfig) { - return remoteApiDelegate() - .makeAsyncCall(environment, packageName, methodName, request, apiConfig); - } - - @Override - public void log(Environment environment, ApiProxy.LogRecord record) { - remoteApiDelegate().log(environment, record); - } - - @Override - public void flushLogs(Environment environment) { - remoteApiDelegate().flushLogs(environment); - } - - @Override - public List getRequestThreads(Environment environment) { - return remoteApiDelegate().getRequestThreads(environment); - } - - @Override - public DevLogService getLogService() { - return new DevLogServiceImpl(); - } - - private static class DevLogServiceImpl implements DevLogService { - private static StreamHandler streamHandler; - - private static synchronized StreamHandler getStreamHandler() { - // TODO: send log message to the log service instead of this. - if (streamHandler == null) { - streamHandler = new ConsoleHandler(); - streamHandler.setFormatter(new Formatter() { - @Override - public String format(LogRecord record) { - return record.getMessage() + "\n"; - } - }); - } - return streamHandler; - } - - @Override - public Handler getLogHandler() { - return getStreamHandler(); - } - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Factory.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Factory.java deleted file mode 100644 index 702c2696d..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Factory.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import com.google.appengine.tools.development.DevAppServer; -import java.io.File; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.Map; - -/** - * Creates new {@link DevAppServer DevAppServers} which can be used to launch - * web applications. - * - */ -class DevAppServer2Factory { - private static final Class[] DEV_APPSERVER_CTOR_ARG_TYPES = {File.class, File.class, - File.class, File.class, String.class, int.class, boolean.class, Map.class}; - - DevAppServer createDevAppServer(final File appDir, final File externalResourceDir, - final File webXmlLocation, final File appEngineWebXmlLocation, final String address, - final int port, final boolean useCustomStreamHandler, final boolean installSecurityManager, - final Map containerConfigProperties, final boolean noJavaAgent) { - return doCreateDevAppServer( - appDir, - externalResourceDir, - webXmlLocation, - appEngineWebXmlLocation, - address, - port, - useCustomStreamHandler, - containerConfigProperties); - } - - private DevAppServer doCreateDevAppServer( - File appDir, - File externalResourceDir, - File webXmlLocation, - File appEngineWebXmlLocation, - String address, - int port, - boolean useCustomStreamHandler, - Map containerConfigProperties) { - - DevAppServer2ClassLoader loader = DevAppServer2ClassLoader.newClassLoader( - getClass().getClassLoader()); - DevAppServer devAppServer; - - try { - Class devAppServerClass = Class.forName(DevAppServer2Impl.class.getName(), false, loader); - - Constructor cons = devAppServerClass.getDeclaredConstructor(DEV_APPSERVER_CTOR_ARG_TYPES); - cons.setAccessible(true); - devAppServer = (DevAppServer) cons.newInstance( - appDir, externalResourceDir, webXmlLocation, appEngineWebXmlLocation, address, port, - useCustomStreamHandler, containerConfigProperties); - } catch (Exception e) { - Throwable t = e; - if (e instanceof InvocationTargetException) { - t = e.getCause(); - } - throw new RuntimeException("Unable to create a DevAppServer", t); - } - return devAppServer; - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Impl.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Impl.java deleted file mode 100644 index c2a9866ad..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/DevAppServer2Impl.java +++ /dev/null @@ -1,425 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import com.google.appengine.tools.development.AppContext; -import com.google.appengine.tools.development.ApplicationConfigurationManager; -import com.google.appengine.tools.development.ContainerService; -import com.google.appengine.tools.development.ContainerUtils; -import com.google.appengine.tools.development.DevAppServer; -import com.google.appengine.tools.development.DevAppServerDatastorePropertyHelper; -import com.google.appengine.tools.development.DevAppServerPortPropertyHelper; -import com.google.appengine.tools.development.EnvironmentVariableChecker.MismatchReportingPolicy; -import com.google.appengine.tools.development.Modules; -import com.google.appengine.tools.development.StreamHandlerFactory; -import com.google.appengine.tools.info.AppengineSdk; -import com.google.appengine.tools.remoteapi.RemoteApiOptions; -import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.api.ApiProxy.Environment; -import com.google.apphosting.utils.config.AppEngineConfigException; -import com.google.apphosting.utils.config.EarHelper; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.net.BindException; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.security.PrivilegedActionException; -import java.security.PrivilegedExceptionAction; -import java.util.HashMap; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.ConsoleHandler; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * {@code DevAppServer} launches a local Jetty server (by default) with a single - * hosted web application. It can be invoked from the command-line by - * providing the path to the directory in which the application resides as the - * only argument. - * - */ -class DevAppServer2Impl implements DevAppServer { - private static final Logger logger = Logger.getLogger(DevAppServer2Impl.class.getName()); - private final ApplicationConfigurationManager applicationConfigurationManager; - private final Modules modules; - private Map serviceProperties = new HashMap(); - private final Map containerConfigProperties; - private final int requestedPort; - private final RemoteApiOptions remoteApiOptions; - private final String webDefaultXml; - - enum ServerState { INITIALIZING, RUNNING, STOPPING, SHUTDOWN } - - /** - * The current state of the server. - */ - private ServerState serverState = ServerState.INITIALIZING; - - /** - * We defer reporting construction time configuration exceptions until - * {@link #start()} for compatibility. - */ - private final AppEngineConfigException configurationException; - - /** - * Used to schedule the graceful shutdown of the server. - */ - private final ScheduledExecutorService shutdownScheduler = Executors.newScheduledThreadPool(1); - - /** - * Latch that we decrement when the server is shutdown or restarted. - * Will be {@code null} until the server is started. - */ - private CountDownLatch shutdownLatch = null; - - /** - * The {@Link ApiProxy.Delegate}. - */ - private DevAppServer2Delegate devAppServer2Delegate; - - /** - * Constructs a development application server that runs the application located in the given - * WAR or EAR directory. - * - * @param appDir The location of the application to run. - * @param externalResourceDir If not {@code null}, a resource directory external to the appDir. - * This will be searched before appDir when looking for resources. - * @param webXmlLocation The location of a file whose format complies with - * http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd. If {@code null}, - * defaults to {@literal /WEB-INF/web.xml} - * @param appEngineWebXmlLocation The name of the app engine config file. If - * {@code null}, defaults to {@literal /WEB-INF/appengine-web.xml} - * @param address The address on which to run - * @param port The port on which to run - * @param useCustomStreamHandler If {@code true} (typical), install {@link StreamHandlerFactory}. - * @param containerConfigProperties Additional properties used in the - * configuration of the specific container implementation. - */ - DevAppServer2Impl(File appDir, File externalResourceDir, File webXmlLocation, - File appEngineWebXmlLocation, String address, int port, boolean useCustomStreamHandler, - Map containerConfigProperties) { - webDefaultXml = - "com/google/appengine/tools/development/devappserver2/webdefault/jetty9/webdefault.xml"; - - String serverInfo = ContainerUtils.getServerInfo(); - if (useCustomStreamHandler) { - StreamHandlerFactory.install(); - } - - String remoteApiHost = (String) containerConfigProperties.get("com.google.appengine.apiHost"); - int remoteApiPort = (Integer) containerConfigProperties.get("com.google.appengine.apiPort"); - this.remoteApiOptions = new RemoteApiOptions() - .server(remoteApiHost, remoteApiPort) - .credentials("test@example.com", "ignoredpassword"); - - requestedPort = port; - ApplicationConfigurationManager tempManager; - AppengineSdk sdk = AppengineSdk.getSdk(); - File schemaFile = new File(sdk.getResourcesDirectory(), "appengine-application.xsd"); - - try { - if (EarHelper.isEar(appDir.getAbsolutePath())) { - tempManager = - ApplicationConfigurationManager.newEarConfigurationManager( - appDir, sdk.getLocalVersion().getRelease(), schemaFile, "dev~"); - String contextRootWarning = - "Ignoring application.xml context-root element, for details see " - + "https://developers.google.com/appengine/docs/java/modules/#config"; - logger.info(contextRootWarning); - } else { - tempManager = - ApplicationConfigurationManager.newWarConfigurationManager( - appDir, - appEngineWebXmlLocation, - webXmlLocation, - externalResourceDir, - sdk.getLocalVersion().getRelease(), - "dev~"); - } - } catch (AppEngineConfigException configurationException) { - modules = null; - applicationConfigurationManager = null; - this.containerConfigProperties = null; - this.configurationException = configurationException; - return; - } - this.applicationConfigurationManager = tempManager; - this.modules = Modules.createModules(applicationConfigurationManager, serverInfo, - externalResourceDir, address, this); - this.containerConfigProperties = ImmutableMap.copyOf(containerConfigProperties); - configurationException = null; - } - - /** - * Sets the properties that will be used by the local services to - * configure themselves. This method must be called before the server - * has been started. - * - * @param properties a, maybe {@code null}, set of properties. - * - * @throws IllegalStateException if the server has already been started. - */ - @Override - public void setServiceProperties(Map properties) { - if (serverState != ServerState.INITIALIZING) { - String msg = "Cannot set service properties after the server has been started."; - throw new IllegalStateException(msg); - } - - if (configurationException == null) { - // Copy the new properties into our own so that we know our map is mutable. - serviceProperties = new ConcurrentHashMap(properties); - serviceProperties.put("appengine.webdefault.xml", webDefaultXml); - if (requestedPort != 0) { - DevAppServerPortPropertyHelper.setPort(modules.getMainModule().getModuleName(), - requestedPort, serviceProperties); - } - DevAppServerDatastorePropertyHelper.setDefaultProperties(serviceProperties); - } - } - - @Override - public Map getServiceProperties() { - return serviceProperties; - } - - /** - * Starts the server. - * - * @throws IllegalStateException If the server has already been started or - * shutdown. - * @throws AppEngineConfigException If no WEB-INF directory can be found or - * WEB-INF/appengine-web.xml does not exist. - * @return a latch that will be decremented to zero when the server is shutdown. - */ - @Override - public CountDownLatch start() throws Exception { - try { - return AccessController.doPrivileged(new PrivilegedExceptionAction() { - @Override public CountDownLatch run() throws Exception { - return doStart(); - } - }); - } catch (PrivilegedActionException e) { - throw e.getException(); - } - } - - private CountDownLatch doStart() throws Exception { - if (serverState != ServerState.INITIALIZING) { - throw new IllegalStateException("Cannot start a server that has already been started."); - } - - reportDeferredConfigurationException(); - - initializeLogging(); - modules.configure(containerConfigProperties); - try { - modules.createConnections(); - } catch (BindException ex) { - System.err.println(); - System.err.println("************************************************"); - System.err.println("Could not open the requested socket: " + ex.getMessage()); - System.err.println("Try overriding --address and/or --port."); - System.exit(2); - } - - devAppServer2Delegate = new DevAppServer2Delegate(remoteApiOptions); - ApiProxy.setDelegate(devAppServer2Delegate); - - TimeZone currentTimeZone = null; - try { - currentTimeZone = setServerTimeZone(); - modules.setApiProxyDelegate(devAppServer2Delegate); - modules.startup(); - } finally { - restoreLocalTimeZone(currentTimeZone); - } - shutdownLatch = new CountDownLatch(1); - serverState = ServerState.RUNNING; - // If you change this please also update - // com.google.watr.client.deployment.DevAppServerDeployment.DevAppServerMonitor. - logger.info("Dev App Server is now running"); - return shutdownLatch; - } - - /** - * Sets the default TimeZone to UTC if no time zone is given by the user via the - * "appengine.user.timezone.impl" property. By calling this method before - * {@link ContainerService#startup()} start}, we set the default TimeZone for the - * DevAppServer and all of its related services. - * - * @return the previous TimeZone - */ - private TimeZone setServerTimeZone() { - // Don't set the TimeZone if the user explicitly set it - String sysTimeZone = serviceProperties.get("appengine.user.timezone.impl"); - if (sysTimeZone != null && sysTimeZone.trim().length() > 0) { - return null; - } - TimeZone utc = TimeZone.getTimeZone("UTC"); - assert utc.getID().equals("UTC") : "Unable to retrieve the UTC TimeZone"; - TimeZone previousZone = TimeZone.getDefault(); - TimeZone.setDefault(utc); - return previousZone; - } - - /** - * Restores the TimeZone to {@code timeZone}. - */ - private void restoreLocalTimeZone(TimeZone timeZone) { - // Don't set the TimeZone if the user explicitly set it - String sysTimeZone = serviceProperties.get("appengine.user.timezone.impl"); - if (sysTimeZone != null && sysTimeZone.trim().length() > 0) { - return; - } - TimeZone.setDefault(timeZone); - } - - @Override - public CountDownLatch restart() throws Exception { - if (serverState != ServerState.RUNNING) { - throw new IllegalStateException("Cannot restart a server that is not currently running."); - } - try { - return AccessController.doPrivileged(new PrivilegedExceptionAction() { - @Override public CountDownLatch run() throws Exception { - modules.shutdown(); - shutdownLatch.countDown(); - modules.createConnections(); - modules.setApiProxyDelegate(devAppServer2Delegate); - modules.startup(); - shutdownLatch = new CountDownLatch(1); - return shutdownLatch; - } - }); - } catch (PrivilegedActionException e) { - throw e.getException(); - } - } - - @Override - public void shutdown() throws Exception { - if (serverState != ServerState.RUNNING) { - throw new IllegalStateException("Cannot shutdown a server that is not currently running."); - } - try { - AccessController.doPrivileged(new PrivilegedExceptionAction() { - @Override public Void run() throws Exception { - modules.shutdown(); - ApiProxy.setDelegate(null); - serverState = ServerState.SHUTDOWN; - shutdownLatch.countDown(); - return null; - } - }); - } catch (PrivilegedActionException e) { - throw e.getException(); - } - } - - @Override - public void gracefulShutdown() throws IllegalStateException { - // TODO: Do an actual graceful shutdown rather than just delaying. - - // Requires a privileged block since this may be invoked from a servlet - // that lives in the user's classloader and may result in the creation of - // a thread. - AccessController.doPrivileged( - new PrivilegedAction>() { - @Override - public Future run() { - return shutdownScheduler.schedule( - new Callable() { - @Override - public Void call() throws Exception { - shutdown(); - return null; - } - }, - 1000, - TimeUnit.MILLISECONDS); - } - }); - } - - @Override - public int getPort() { - reportDeferredConfigurationException(); - return modules.getMainModule().getMainContainer().getPort(); - } - - protected void reportDeferredConfigurationException() { - if (configurationException != null) { - throw new AppEngineConfigException("Invalid configuration", configurationException); - } - } - - @Override - public AppContext getAppContext() { - reportDeferredConfigurationException(); - return modules.getMainModule().getMainContainer().getAppContext(); - } - - @Override - public AppContext getCurrentAppContext() { - AppContext result = null; - Environment env = ApiProxy.getCurrentEnvironment(); - //Some tests create environments with null version id's - if (env != null && env.getVersionId() != null) { - String moduleName = env.getModuleId(); - result = modules.getModule(moduleName).getMainContainer().getAppContext(); - } - return result; - } - - @Override - public void setThrowOnEnvironmentVariableMismatch(boolean throwOnMismatch) { - if (configurationException == null) { - applicationConfigurationManager.setEnvironmentVariableMismatchReportingPolicy( - throwOnMismatch ? MismatchReportingPolicy.EXCEPTION : MismatchReportingPolicy.LOG); - } - } - - /** - * We're happy with the default logging behavior, which is to - * install a {@link ConsoleHandler} at the root level. The only - * issue is that we want its level to be FINEST to be consistent - * with our runtime environment. - * - *

Note that this does not mean that any fine messages will be - * logged by default -- each Logger still defaults to INFO. - * However, it is sufficient to call {@link Logger#setLevel(Level)} - * to adjust the level. - */ - private void initializeLogging() { - for (Handler handler : Logger.getLogger("").getHandlers()) { - if (handler instanceof ConsoleHandler) { - handler.setLevel(Level.FINEST); - } - } - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteApiDelegate.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteApiDelegate.java deleted file mode 100644 index d4d9803f7..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteApiDelegate.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import com.google.appengine.tools.remoteapi.RemoteApiOptions; -import com.google.apphosting.api.ApiProxy; -import com.google.apphosting.api.ApiProxy.ApiConfig; -import com.google.apphosting.api.ApiProxy.Delegate; -import com.google.apphosting.api.ApiProxy.Environment; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * {@link ApiProxy.Delegate} implementation that forwards API requests to an API server over HTTP. - * This implementation is not thread-safe so every thread that needs to make API calls must have - * its own instance. - * - */ -class RemoteApiDelegate implements Delegate { - private static final Logger logger = Logger.getLogger(RemoteApiDelegate.class.getName()); - - private final ExecutorService executor; - private final RemoteRpc remoteRpc; - - RemoteApiDelegate(RemoteRpc rpc, RemoteApiOptions options) { - this.executor = Executors.newFixedThreadPool(options.getMaxConcurrentRequests()); - this.remoteRpc = new RemoteRpc(options); - } - - @Override - public byte[] makeSyncCall(Environment env, String serviceName, String methodName, - byte[] request) { - String requestId = RequestIdFilter.threadRequestId(); - if (requestId == null) { - requestId = "no-request-id"; - } - return remoteRpc.call(serviceName, methodName, requestId, request); - } - - @Override - public Future makeAsyncCall(final Environment env, final String serviceName, - final String methodName, final byte[] request, ApiConfig apiConfig) { - // TODO respect deadline in apiConfig - return executor.submit( - new Callable() { - @Override - public byte[] call() throws Exception { - // note that any exceptions thrown will be captured and thrown by the Future instead. - return makeSyncCall(env, serviceName, methodName, request); - } - }); - } - - @Override - public void log(Environment environment, ApiProxy.LogRecord record) { - logger.log(toJavaLevel(record.getLevel()), - "[" + record.getTimestamp() + "] " + record.getMessage()); - } - - @Override - public List getRequestThreads(Environment environment) { - return Collections.emptyList(); // not implemented - } - - @Override - public void flushLogs(Environment environment) { - // not implemented - } - - private static Level toJavaLevel(ApiProxy.LogRecord.Level apiProxyLevel) { - switch (apiProxyLevel) { - case debug: - return Level.FINE; - case info: - return Level.INFO; - case warn: - return Level.WARNING; - case error: - return Level.SEVERE; - case fatal: - return Level.SEVERE; - default: - return Level.WARNING; - } - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteRpc.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteRpc.java deleted file mode 100644 index 3b45718bc..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RemoteRpc.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import com.google.appengine.tools.remoteapi.RemoteApiException; -import com.google.appengine.tools.remoteapi.RemoteApiOptions; -import com.google.apphosting.base.protos.api.RemoteApiPb; -import com.google.protobuf.ByteString; -import com.google.protobuf.ExtensionRegistry; -// -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.util.logging.Level; -import java.util.logging.Logger; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.HttpClientBuilder; - -/** - * An RPC transport that sends protocol buffers over HTTP. This class is not thread-safe, in that - * two different threads cannot use it to send requests at the same time. - */ -class RemoteRpc { - private static final Logger logger = Logger.getLogger(RemoteRpc.class.getName()); - - private final RemoteApiOptions options; - private final HttpClient httpClient; // This is why the class isn't thread-safe. - - private int rpcCount = 0; - - RemoteRpc(RemoteApiOptions options) { - this.options = options; - this.httpClient = HttpClientBuilder.create().disableRedirectHandling().build(); - } - - /** - * Makes an RPC call using the injected RemoteRpc instance. Logs how long it took and - * any exceptions. - * @throws RemoteApiException if the RPC fails. - * @throws RuntimeException if the server threw a Java runtime exception - */ - byte[] call(String serviceName, String methodName, String requestId, byte[] request) { - logger.log(Level.FINE, "remote API call: {0}.{1}", new Object[] {serviceName, methodName}); - - long startTime = System.currentTimeMillis(); - try { - - RemoteApiPb.Request requestProto = makeRequest(serviceName, methodName, requestId, request); - RemoteApiPb.Response responseProto = callImpl(requestProto); - - if (responseProto.hasJavaException()) { - // We don't expect this at all from the devappserver2 API server, so handle it minimally. - logger.fine("remote API call: failed due to a server-side Java exception"); - Throwable exception = parseJavaException(responseProto, - requestProto.getServiceName(), requestProto.getMethod()); - throw new RemoteApiException("response was an exception", - requestProto.getServiceName(), requestProto.getMethod(), exception); - } else if (responseProto.hasException()) { - String pickle = responseProto.getException().toString(); - - logger.log(Level.FINE, - "remote API call: failed due to a server-side Python exception:\n{0}", pickle); - throw new RemoteApiException("response was a python exception:\n" + pickle, - requestProto.getServiceName(), requestProto.getMethod(), null); - } - - return responseProto.getResponse().toByteArray(); - - } finally { - long elapsedTime = System.currentTimeMillis() - startTime; - logger.log(Level.FINE, "remote API call: took {0} ms", elapsedTime); - } - } - - RemoteApiPb.Response callImpl(RemoteApiPb.Request requestProto) { - rpcCount++; - - byte[] requestBytes = requestProto.toByteArray(); - String url = "http://" + options.getHostname() + ":" + options.getPort() - + options.getRemoteApiPath(); - HttpPost post = new HttpPost(url); - post.addHeader("Host", options.getHostname()); - post.addHeader("X-appcfg-api-version", "1"); - post.addHeader("Content-Type", "application/octet-stream"); - post.setEntity(new ByteArrayEntity(requestBytes)); - try { - HttpResponse response = httpClient.execute(post); - if (response.getStatusLine().getStatusCode() != 200) { - throw makeException( - "unexpected HTTP response: " + response.getStatusLine(), null, requestProto); - } - int max = options.getMaxHttpResponseSize(); - byte[] buf = new byte[65536]; - byte[] body; - try (InputStream in = response.getEntity().getContent(); - ByteArrayOutputStream bout = new ByteArrayOutputStream()) { - int n; - while (bout.size() < max - && (n = in.read(buf, 0, Math.min(buf.length, max - bout.size()))) > 0) { - bout.write(buf, 0, n); - } - body = bout.toByteArray(); - } - return RemoteApiPb.Response.parseFrom(body, ExtensionRegistry.getEmptyRegistry()); - - } catch (IOException e) { - throw makeException("I/O error", e, requestProto); - } - } - - void resetRpcCount() { - rpcCount = 0; - } - - int getRpcCount() { - return rpcCount; - } - - private static RemoteApiPb.Request makeRequest( - String packageName, String methodName, String requestId, byte[] payload) { - return RemoteApiPb.Request.newBuilder() - .setServiceName(packageName) - .setMethod(methodName) - .setRequest(ByteString.copyFrom(payload)) - .setRequestId(requestId) - .build(); - } - - // - private static Throwable parseJavaException( - RemoteApiPb.Response parsedResponse, String packageName, String methodName) { - try { - InputStream ins = parsedResponse.getJavaException().newInput(); - ObjectInputStream in = new ObjectInputStream(ins); - return (Throwable) in.readObject(); - } catch (IOException | ClassNotFoundException e) { - throw new RemoteApiException( - "remote API call: " + "can't deserialize server-side exception", packageName, methodName, - e); - } - } - - private static RemoteApiException makeException(String message, Throwable cause, - RemoteApiPb.Request request) { - logger.log(Level.FINE, "remote API call: {0}", message); - return new RemoteApiException("remote API call: " + message, - request.getServiceName(), request.getMethod(), cause); - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RequestIdFilter.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RequestIdFilter.java deleted file mode 100644 index b0dcc63aa..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/RequestIdFilter.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import java.io.IOException; -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; - -/** - * A Filter that makes the X-Appengine-Dev-Request-Id header available within the request thread. - * - */ -public class RequestIdFilter implements Filter { - private static final ThreadLocal threadRequestId = new ThreadLocal<>(); - - @Override public void init(FilterConfig filterConfig) {} - - @Override public void destroy() {} - - @Override public void doFilter( - ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - if (request instanceof HttpServletRequest) { - HttpServletRequest httpRequest = (HttpServletRequest) request; - String requestId = httpRequest.getHeader("X-Appengine-Dev-Request-Id"); - threadRequestId.set(requestId); - } - try { - chain.doFilter(request, response); - } finally { - threadRequestId.remove(); - } - } - - static String threadRequestId() { - return threadRequestId.get(); - } -} diff --git a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/StandaloneInstance.java b/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/StandaloneInstance.java deleted file mode 100644 index 2234105cd..000000000 --- a/lib/tools_api/src/main/java/com/google/appengine/tools/development/devappserver2/StandaloneInstance.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.appengine.tools.development.devappserver2; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.google.appengine.tools.development.DevAppServer; -import com.google.appengine.tools.development.SharedMain; -import com.google.appengine.tools.development.proto.Config; -import com.google.appengine.tools.util.Action; -import com.google.appengine.tools.util.Option; -import com.google.appengine.tools.util.Parser; -import com.google.appengine.tools.util.Parser.ParseResult; -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.io.PrintWriter; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * A runnable program that acts as an instance that can be run from within the - * Python devappserver2. See apphosting/tools/devappserver2/java_runtime.py. - * - */ -public class StandaloneInstance extends SharedMain { - private static final String SDK_ROOT_ENVIRONMENT_VAR = "SDKROOT"; - private static final String JAVA_PATH_PREFIX = "google/appengine/tools/java/"; - private static final String JAVA_LIB_PATH = JAVA_PATH_PREFIX + "lib"; - - private static final Logger logger = Logger.getLogger(StandaloneInstance.class.getName()); - - private final Action startAction = new StartAction(); - private final File sdkRoot; - private final File javaLibs; - - public static void main(String[] args) throws Exception { - SharedMain.sharedInit(); - System.setProperty("com.google.appengine.devappserver2", "true"); - new StandaloneInstance().run(args); - } - - private StandaloneInstance() { - String sdkRootPath = System.getenv(SDK_ROOT_ENVIRONMENT_VAR); - if (sdkRootPath == null) { - logger.severe("Environment does not have SDKROOT variable set"); - System.exit(1); - } - sdkRoot = new File(sdkRootPath); - - javaLibs = new File(sdkRoot, JAVA_LIB_PATH); - if (!javaLibs.isDirectory()) { - logger.log(Level.SEVERE, "Java libraries directory does not exist or cannot be read: {0}", - javaLibs); - System.exit(1); - } - } - - private void run(String[] args) throws Exception { - Parser parser = new Parser(); - ParseResult result = parser.parseArgs(startAction, buildOptions(), args); - result.applyArgs(); - } - - class StartAction extends Action { - StartAction() { - super("start"); - } - - @Override - public void apply() { - List args = getArgs(); - if (args.size() != 2) { - printHelp(System.err); - System.exit(1); - } - try { - apply(args); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void apply(List args) throws IOException { - File inputConfigFile = new File(args.get(0)); - Config inputConfig; - try (InputStream in = new FileInputStream(inputConfigFile)) { - inputConfig = Config.parseFrom(in); - } finally { - boolean deleted = inputConfigFile.delete(); - if (!deleted) { - logger.log(Level.WARNING, "Could not delete {0}", inputConfigFile); - } - } - String apiHost = inputConfig.getApiHost(); - int apiPort = inputConfig.getApiPort(); - String applicationRoot = inputConfig.getApplicationRoot().toStringUtf8(); - File appDir = new File(applicationRoot); - validateWarPath(appDir); - configureRuntime(appDir); - - boolean noJavaAgent = true; - // TODO: Implement Java Agent support. - - Map containerConfigOptions = - ImmutableMap.of( - "com.google.appengine.apiHost", apiHost, - "com.google.appengine.apiPort", apiPort); - DevAppServer2Factory devAppServer2Factory = new DevAppServer2Factory(); - File externalResourceDir = null; - File webXmlLocation = null; - File appEngineWebXmlLocation = null; - // TODO: figure out if we need to supply values for these, and delete the parameters - // if not. - boolean useCustomStreamHandler = true; - boolean installSecurityManager = false; - DevAppServer server = devAppServer2Factory.createDevAppServer( - appDir, externalResourceDir, webXmlLocation, appEngineWebXmlLocation, "localhost", 0, - useCustomStreamHandler, installSecurityManager, containerConfigOptions, noJavaAgent); - Map stringProperties = getSystemProperties(); - postServerActions(stringProperties); - addPropertyOptionToProperties(stringProperties); - server.setServiceProperties(stringProperties); - CountDownLatch serverLatch; - try { - serverLatch = server.start(); - int port = server.getPort(); - File outputPortFile = new File(args.get(1)); - try (PrintWriter printWriter = new PrintWriter(outputPortFile, UTF_8.name())) { - printWriter.println(port); - } - serverLatch.await(); - } catch (Exception e) { - logger.log(Level.SEVERE, "Failed to start server", e); - } - } - } - - @Override - protected void printHelp(PrintStream out) { - out.println("Usage: [options] "); - out.println(" where is the file to which the HTTP server port is written"); - out.println(""); - out.println("Options:"); - for (Option option : buildOptions()) { - for (String helpString : option.getHelpLines()) { - out.println(helpString); - } - } - } - - private List